{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import torch as t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        \n",
    "        self.fc1 = nn.Linear(28*28, 300)\n",
    "        self.fc2 = nn.Linear(300, 10)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.view(x.shape[0], -1)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.logsigmoid(self.fc2(x))\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mnist_loader import load_data_shared, vectorized_result\n",
    "training_data, validation_data, test_data = load_data_shared(filename=\"../mnist.pkl.gz\",\n",
    "                                                                     seed=666,\n",
    "                                                                     train_size=1000,\n",
    "                                                                     vali_size=0,\n",
    "                                                                     test_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(data, net):\n",
    "    with t.no_grad():\n",
    "        #for index in range(test_data[0].shape[0]):\n",
    "            # get the inputs\n",
    "        inputs, labels = t.Tensor(data[0]), data[1]\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        _, predicted = t.max(outputs, 1)\n",
    "\n",
    "        correct = (predicted == t.Tensor(labels)).sum().item()\n",
    "        accuracy = correct / data[0].shape[0]\n",
    "        return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit(net, criterion, optimizer):\n",
    "    loss_scores = []\n",
    "    test_scores = []\n",
    "    train_scores = []\n",
    "    for epoch in range(100):  # loop over the dataset multiple times\n",
    "\n",
    "        # get the inputs\n",
    "        inputs, labels = t.Tensor(training_data[0]), t.Tensor(training_data[1])\n",
    "        vector_labels = t.Tensor([vectorized_result(y) for y in training_data[1]])\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels.long())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        loss_scores.append(loss.item())\n",
    "        train_scores.append(predict(training_data, net))\n",
    "        test_scores.append(predict(test_data, net))\n",
    "    print('Finished Training')\n",
    "    return loss_scores, train_scores, test_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ReLU + Sigmoid + Cross Entropy + L2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net1 = Net()\n",
    "criterion1 = nn.CrossEntropyLoss()\n",
    "optimizer1 = optim.SGD(net1.parameters(), lr = 1e-1, weight_decay=1e-2)\n",
    "loss_scores1, train_scores1, test_scores1 = fit(net1, criterion1, optimizer1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXxU1fnH8c+ThCSQhC0JARJCWMImIkIE3BVBcUWtC1jXolTrVm21ttpqtZva1tpfaRWXgqggbohKxX0HTAAFwk5YsgAJCUlIyDrz/P64A4QQyARmMsnkeb9evJJ775mZ5zLwzc2Zc88RVcUYY0zrFxLoAowxxviGBboxxgQJC3RjjAkSFujGGBMkLNCNMSZIhAXqhePi4jQlJSVQL2+MMa3S0qVLd6lqfEPHAhboKSkpZGRkBOrljTGmVRKRrYc7Zl0uxhgTJCzQjTEmSFigG2NMkLBAN8aYIGGBbowxQcIC3RhjgoQFujHGBImAjUM3xphWqXATrHwd3K6jf46BEyBxpO9q8rBAN8YYb2V9AXOvh8piQJr00LorT5SFxxFjgW6MMQGydCa8fy/EpsJPv4Quvfcfqq51s3tvNbvKqigqr6awrJr8PZVs2FnG+vwysvLL2FNVu7/9H8KGcq0fSrRAN8YEj+0/wNu3QdnOJj9UAVXFreBWRQ/66ibaVUJmh1H8t+PvKH23gKLyXIrKnRAvraxt8DnjosNJ7RbDZSMSSegYSVx0OF2jIjiuZ8djPNGGWaAbY4LDmvfgrVugfRcYMvGgQ1UuNwWlVZRV1VJZ46Kq1kVljdv5U+uiqsZFVa27wacVgYiwUHaF9+TNkItx57toF7KXrlHhDO7ZkdiocGKjIoiNDne+j3a+j4uKoFOHds1x5vtZoBtjHG4XbPoMasoDXUnT7VgJX/4Vd88TWTx6GlsqoykqryJ/TxXLtxWTmVeC29OJLQKd27cjLjqC2C5OGHeNCncCOTrCE9DOdteoCDq3b0dIiNNfflsAT9EbFujGGKgshTenwIYPA13JUVsecza35E5h1+zN+/fFRIQxuGdH7hybypi+saQmRNOlQzihIU37QLO1sEA3pq3bvRVmT4KCdTDhcehzeqArApz+7PyyavJ2V7B7bzUlFTUU7/taUUNReTXZRRXsra6lhjB20YvzhvbggmE9GNQ9hq5R4USEhQb6NJqVBboxbVlRFrxwLtRWw7VvQr+z/f6SBXuqWJxVyOKsQvKKK+jq6X+ODAuhaK8zQmR7SSUb88soqzr4w8bQkBC6RsUQGxVOXHQEx6dEMSAhhgEJMQzv1ZnwsLZ9r6QFujFt2aJ/Q2UJ3Po1xA/06VOrKnuqaikqq2b19lIWbXJCfEN+GQDREWH0ju3A+p1l7CqrotrlpnP7dsRGR9AtJoLLRyQyICGGfvHRxMdEEBcdTsfIA/3Z5lAW6Ma0VdV7YcVcZ0TIUYb5rrIqMvNK2bBzDxt2lrG9tJLCOmOxq10HRo50CA/lpJSuXD4iiZP7xTK0Z0fCQp0r6n3DBYO1b7u5WKAb01atfgeqSmDEDU16WMGeKj5YtZ33Vmznuy1FqGf0SFx0OIldOpDQMZLBPTru7xaJjQ6nd2wUw5I60S604S4RESHUsvyYWaAb01YtnQFd+0HKaY02LSyrYmHmTt5bkcfirELcCv27RXPX2FRO7hfLgATnQ0gTWBboxrRF+WshezGMf9QZmF1PSUUN320u2v/h5ertpahC37gobj+7PxcN68mAhGikgceawPEq0EVkAvA0EAo8r6p/qXc8GZgJdPa0eUBVF/i4VmOMryx7CULawQnX7N9VWFbFc19t5puNu/bfiBMeFsLI5C7cM24A44ckMKh7jIV4C9ZooItIKDANGA/kAOkiMl9VV9dp9hAwV1X/IyJDgAVAih/qNcYcq9oq+GE2DLoQouMB+N/K7Tw0bxUlFTWM6N2FOz1dKcN7dSayXdsay92aeXOFPgrYqKpZACIyB5gI1A10BfbNNtMJyPNlkcYYH1GFLx6HiiIYeQPZRXt5YuE63v0hj+MTOzF76hgGJMQEukpzlLwJ9EQgu852DjC6XptHgA9F5E4gChjX0BOJyFRgKkBycnJTazXGHIvaKph/J6x4jZ0pE3noqyg+XvcZYSHCL8YP4Naz+h12FIppHbwJ9IY6zLTe9mRghqr+TUROBmaJyFBVPWj6MlWdDkwHSEtLq/8cxhhfKiuAjR+z779rbfoMwnKX8EL4j3ls7QXERe/hzrGpXDMqme6dIgNbq/EJbwI9B+hVZzuJQ7tUpgATAFR1kYhEAnFAvi+KNMZ4R1Up3lsDQNR79xG+9u39x2q1HXfX3MX2bhN4+oIUzh/ao83fKh9svAn0dCBVRPoAucAk4Jp6bbYB5wAzRGQwEAkU+LJQY8yRlVXVcuuspXy9cRddKWVRxLu86hrLv12X0LNTJEP6pXDbKUMYmtgp0KUaP2k00FW1VkTuABbiDEl8UVUzReRRIENV5wO/AJ4TkXtwfr+7UVWtS8WYZlJYVsVNM9LJzCvlrrH9Ob1gDhEbaokfdzdzThhFUpcOgS7RNAMJVO6mpaVpRkZGQF7bmGCSV1zBdS8sIWd3Bf/+8QjOGdQN/nWSs3LPzR8FujzjYyKyVFXTGjpmHWjGtGK5xRVcPX0R+aVVvPSTUZwzOAG2LYLCDTDyxkCXZ5qZ3fpvTCuVV1zB5OmLKd5bw8s3j+aEXp2dA0tnQkRHOO7SwBZomp1doRvTCu0oqWTyc4vZXV7NrCl1wrxiN6yeB8dfCeFRgS3SNDu7QjemlVm+bTd3vLqckooaXpoyiuH7whyc+c1rK2Fk06bENcHBrtCNaSVUlf9+s5mrnl2ECMy+ZQwjkrscaFBZCl/9DXqNhh4nBK5QEzB2hW5MK1BYVsVv3l7JwsydjBucwN+uPIFOHdod3OiLx6EsHybPDkyRJuAs0I1p4fbNhLinspYHLxjMzaf3OXQK2/y1sOQZGHEdJI4MTKEm4CzQjWmhKqpd/Obtlby9PJfjEzvxt6tOaHgmRFX43/3Oh6DnPNz8hZoWwwLdmBYov7SSW17KYEVuCT8fl8rtZ/c/eCbE3GWQu9T5vngbbP4CLvgrRMUFpmDTIligG9PCZOaVcPPMDEoqaph+XRrjhyQc3CD9BVhwH6jrwL5eY2DkTc1bqGlxLNCNaUFW5pQw+bnFxESG8catpzCkZ8cDB90uWPggLPkPpJ4HFz0FYRHOsfZdIMRWFmrrLNCNaSE27yrnxv9+R6f27XjztlMOnqO8shTenAIbPoQxt8O5j1mAm0NYoBvTAuwsreS6F5YAMGvKqIPDfPdWmD0Jdq13rsrTfhKgKk1LZ4FuTIBtKijjtpeXsru8mjlTT6ZvfPSBg9nfwZxrwFUN174Jfc8KVJmmFbBANyZAXG7nzs8nF64jsl0oz92QxvFJdRafWPE6vHM7dEqEa+ZCXGrgijWtggW6MQGwp7KGKTMz+G5zEeMGd+NPlx1Pt46ebha3Gz7/M3z5BPQ+Da6eBR26BrZg0ypYoBvTzFSV37y9iqVbd/PkFcO4YmTSgTs/VWHerbDiNTjxWrjwKQgLD2zBptXwanIuEZkgIutEZKOIPNDA8adE5HvPn/UiUuz7Uo0JDq+lZ/PuD3ncO34AV6b1Ovg2/txlTpifdg9c8i8Lc9MkjV6hi0goMA0YD+QA6SIyX1VX72ujqvfUaX8ncKIfajWm1Vu3Yw8Pz8/k9NQ4bjuz36ENls2Adh3gtHuh/nwtxjTCmyv0UcBGVc1S1WpgDjDxCO0nAzbdmzH1lFXVcvury4iJbMffrxpOSEi9wK7aAyvfhKGXQ2THhp/EmCPwJtATgew62zmefYcQkd5AH+DTwxyfKiIZIpJRUFDQ1FqNabXKq2q58cXv2LyrnH9cPZz4mIhDG616E2rKYcSNzV6fCQ7eBHpDv/fpYdpOAt5QrTvJRJ0HqU5X1TRVTYuPj/e2RmNatb3Vtdw0I53l2cX8c9KJnJZ6mAm0ls6AbkMgqcEF3Y1plDeBngP0qrOdBOQdpu0krLvFmP0qa1z8ZEY6GVuK+MfVw7lwWI+GG25fAXnLYeSN1ndujpo3gZ4OpIpIHxEJxwnt+fUbichAoAuwyLclGtN6PfXxehZnFfHU1cO5+ISeh2+4bCaERcKwq5qvOBN0Gg10Va0F7gAWAmuAuaqaKSKPisgldZpOBuao6uG6Y4xpU9ZsL+X5rzYz6aReTBze4MdOjsoS567QIROdWRONOUpe3VikqguABfX2/a7e9iO+K8uY1s3tVn791ko6t2/HA+cPOnLjz/8CVaVw8u3NU5wJWl7dWGSMaZpXvtvG99nF/PaiIXTucISbg3ZmwpJnIe0m6HFC8xVogpIFujE+tqOkkif+t5bT+scxcfgR+s1VYcH9zpjzsb9tvgJN0LK5XIzxodLKGm6akY5LlT9cOvTg2/rrW/UmbP3amePcJt8yPmBX6Mb4SGWNi5tnZrAxfw/PXDuSlLiowzeu3gsf/tbpZhlxQ/MVaYKaXaEb4wO1Ljd3zV7Od5uLeHrScM4Y0MiNc5lvwZ48uPxZW0rO+IwFujHHqKi8mnte+54v1hfw8MVDjjxEcZ+lMyFuAKSc7v8CTZthgW7MMVi6dTd3vLqMwrJq/njZUH48unfjD9q5GnK+g3P/aHeFGp+yQDfmKM1bnssvX/+BHp0jeetnpzA0sVPjDwLnrtDQcDhhsn8LNG2OBboxR2FxViH3vfEDJ6V05ZnrRtKpfTvvHlhTCT/MgUEXQVSsf4s0bY6NcjGmiTbvKufWl5fSOzaqaWEOsGY+VBbDSBvZYnzPAt2YJijeW81PZqQTIsKLN5zUtDAHZ4rcLn0g5Qy/1GfaNgt0Y7yUV1zB5OeWkLu7gunXjSQ5tkPTnmBnJmz9BkZcDyH2X8/4nvWhG+OFH7KLufmlDCqrXTx3QxppKU28s1MVPngAIjvZjUTGbyzQjWnE+yu2c+/c74mPieCVm0czICGm6U+yeh5s/hIu+Kt9GGr8xgLdmMNQVf716Ub+9tF6RvbuwvTrRhIb3cBaoI2pLoeFD0L34yHtJ74v1BgPC3RjGlBZ4+KBN1cw7/s8Lj8xkT//6Hgiwo7yFv2v/galuXDFi3abv/ErrwJdRCYATwOhwPOq+pcG2lwFPIKzgPQPqnqND+s0ptlU1ri46b/pLMoq5L7zBvKzs/odedZEgPULYckz4K499Ni2xTBsEiSP8U/Bxng0GugiEgpMA8bjLBidLiLzVXV1nTapwK+BU1V1t4h081fBxviTy63cPWc5i7IK+ftVJ3D5iKQjP0AVFk2DDx+CzsnQsYH5z1PPhXMf80/BxtThzRX6KGCjqmYBiMgcYCKwuk6bW4BpqrobQFXzfV2oMf6mqjw0byULM3fy8MVDGg/z2mpY8AtY9pKzHuilz0B4E4cyGuND3gyGTQSy62znePbVNQAYICLfiMhiTxeNMa2GqvKXD9Yy+7ts7ji7Pzed2ufID9hbBC9f7oT56b+EK2ZYmJuA8+YKvaHOQ23geVKBs4Ak4CsRGaqqxQc9kchUYCpAcnJyk4s1xh8qa1zc/8YK5v+Qx49HJ/OLcwcc+QG7NsKrV0FJNlz2LJwwqXkKNaYR3gR6DtCrznYSkNdAm8WqWgNsFpF1OAGfXreRqk4HpgOkpaXV/6FgTLMr2FPF1FkZLN9WzP0TBnLbmQ18ALrpU9jwkfO9up3JtUJC4YZ37YNO06J4E+jpQKqI9AFygUlA/REs84DJwAwRicPpgsnyZaHG+NrmXeVc98ISdpVV8cy1I5gwtMfBDVTh2/+Dj34HYZEQ6pm3JS7VGYLYJaXZazbmSBoNdFWtFZE7gIU4wxZfVNVMEXkUyFDV+Z5j54rIasAF3Keqhf4s3JhjkZlXwg0vfodbYe5PT2ZYUueDG9RWw/v3wvJZMORSuOwZaNc+MMUa4yVRDUzPR1pammZkZATktU3bpaos2lTIT2ctJSYyjJemjKZ/t2jnavz1G5zx5ABuF7hr4Iz74axf22RapsUQkaWqmtbQMbtT1AQ9VeX9ldv5ZE0+i7MK2V5SSd+4KGbdPJrEzp6r7hVzYfU7cPxV0NHT9ZJ8Mgw8P3CFG9NEFugmqFXVuvjNW6t4c1kOcdHhjO4by5i+sVwyrCedOnj6xCtL4aPfQuJIZ9SKXY2bVsoC3QStovJqbp21lO+2FHH3OancfU4qISENjML94nEoy4fJsy3MTatmgW6C0pfrC3hw3kp2llbx9KThTBxe/144j/w1sPg/zqITiSObt0hjfMwC3QSV7KK9PPbeaj5cvZOU2A7MmTqGEcldGm5ctQfevRsiYuCch5u3UGP8wALdBAVVZU56No/MzyREhPsnDGTKaX0OP+Vt8TZ4dRIUrIXLp9uiEyYoWKCbVm9vdS0Pvb2Kt5bncnpqHE9cMYwenY4wZjw7HeZMdsaaX/sG9BvbfMUa40cW6KbVUlW+2VjI79/NZGNBGfeOH8DtZ/cntKEPPvcpzYNZlzlX5De+D/EDm69gY/zMAt20Oi63sjirkH98vJ70Lbvp2SmSl6eM5tT+cY0/+KPfgasarnsbuvb1f7HGNCMLdNMqpG8p4pXFW1m3s4xNBWVU17rp3jGSxy4dylVpSd4tD7flG1j5unP3p4W5CUIW6KZF21layZ8XrGHe93l0jQpnWFInTk+N47ieHTnvuO5EtvNyjU5XLSy4Dzr1gtPu8W/RxgSIBbppkbKL9vLKkm3MWrSFGpdyx9n9+dnZ/egQfpT/ZDNegPxMuGqWLURhgpYFuvGbyhoXm3eV41YlNiqCrlHhlFXVsn7nHjbs3EPO7goKy6spLKui1q10jQonNiqCbUV7+XTtTgDOH9qD+ycMpHds1NEXsvkr+OQx6Hs2DL7YR2dnTMtjgW4OsaeyhtziCmIi2xEbFU5EWAi5xRVs2FlG1q5yal1uANwKJRU1FJZVUVReTY3bmblTVcndXcGWwnLcR5jMMzwshNiocGKjwwkNCWFr4V4Ky6poHx7Gz87qzzWjk+nZ+RinrF02C977udNnfsk/of7iFcYEEQv0Nqqq1sXybcWsyi3Zf5W8o7SKjTv3kFdSeVDbdqFCjavhZA4PDSE2OpwuHcIJDzswD0pqQjQXDutBakIM4aEhFJZXUVRWTfvwUFITYhiQEE33jpGHrg6E8wOhof1N4nbBx4/At/90rsyvnAHtOzf2KGNaNQv0IFJZ42LJ5iIWbSpkcVYhVbVuRvfpypi+sfTsHMmGnWWsz9/DiuwSlm3bTVWtc6XdLlToGhVOfEwEo/vGkpoQTXLXDpRV1lJYXk1pZQ29unRgQEIM/eKjaB/ufBApCJHtQo49fOs55uerKoO3psK69yFtCpz/+IHVhowJYhborcS+hUgOF3Zfri/gV2+uYHtJJWEhwvBenYmOCmdO+jZmfLtlf7vw0BBSE6L58ejenNwvlhHJnekaFe7zUA6Ykhznlv78TDj/CRg11bpZTJvhVaCLyATgaZwl6J5X1b/UO34j8CTOmqMA/1LV531YZ5ulqryekcMfF6yhZ+f2/HxcKucOSdgfwGVVtfzx/TXM/m4b/eKjeOGGNE7uF7t/NEh1rZsfcoopLKuif7cYUmI7EBYapFPE5i6F2ZOhei9cMxdSxwe6ImOaVaOBLiKhwDRgPJADpIvIfFVdXa/pa6p6hx9qbLN2lFTy67dW8Nm6AtJ6d6GwvJqfzlrKcT07khIXxYade9i8q5xat/LTM/pyz/gBh4zLDg8L4aSUrgE6g2aU+Ta8fStEd4Pr5kHCkEBXZEyz8+YKfRSwUVWzAERkDjARqB/oxkdqXW5eWrSVpz5eT43LzSMXD+H6k1Nwq/LO93n854tNrMgpZkC3GMYOSmDC0O4M79WGPvBThR0roLrc2c76Ar74C/QaDVe/AtHxga3PmADxJtATgew62znA6Aba/UhEzgDWA/eoanYDbcwR7Jts6tH3Mlm/s4zTU+N4dOJQ+sQ5Y7BDEH40MokfjUwKcKUBtvjfsPA3B+8bdjVc/E9oFxmYmoxpAbwJ9IY+Uao/hu1dYLaqVonIrcBM4JA5SUVkKjAVIDk5uYmlBq8120t55/s83l+ZR3ZRBUld2vPsdSMP6is3Hnt2wGd/cqa8PfVuZ194tLPakP1dmTbOm0DPAXrV2U4C8uo2UNXCOpvPAY839ESqOh2YDpCWlnaEW07ajsVZhUx+bjEhIpzaP447z07lkuE9vZ+jJNhs+AhyMuD0X0BY+KHH982WeOHfbIItY+rxJtDTgVQR6YMzimUScE3dBiLSQ1W3ezYvAdb4tMogpar8ecEaenSM5N07TyM2OiLQJQWOqnMT0EcPAwpbvnLmXam7ktDWb2HFa3D6Ly3MjWlAo+PXVLUWuANYiBPUc1U1U0QeFZFLPM3uEpFMEfkBuAu40V8FB5MFK3fwQ04J94wf0LbDvLYa5t/hXH0fdylM/Ldzlf78WNixEipLoGK3M1tixyQ4/d5AV2xMiyT7blhpbmlpaZqRkRGQ124Jalxuxv/9C8LDQvjf3WcceZWdYLa3CF67DrZ+DWf+Cs58AEJCDiwTV15wcPsrZzqhb0wbJSJLVTWtoWN2p2iAzEnPZkvhXl64Ia3thnnBenj1KmdZuMufg2FXHTjW6ySY+gWseRfU5ezr1MtmSzTmCCzQA6C8qpanP97AqJSujB3ULdDl+NeGj5xuk/pcNbB4GoSGw43vQa9Rh7bplAhjbvV/jcYECQv0APj9u5kUllcx/fqRwTss0e2GTx+Fr586fJvuxzs3AnXp3Xx1GRPELNCb2dvLc5ibkcOdY/szIrlLoMvxj+pyePunTnfJyBvh3D9CSAP/1MIibOy4MT5kgd6MsgrKePDtVYxK6crd56T698W+/gekH2Z+tM694dJ/H3xl/P2r8OWTTleIt0LC4JQ74aQpB/aV5jkTZO1YAef9GcbcZqFtTDOxQG8mlTUubn91ORFhITw9ebh/ZzzcsRI++T0knQSx/Q8+purME/7cWJg8GxLTnLbf/MP5Pn6g969TuAnevxd2rXeuwneugtmToGoPTJ4DA87z7XkZY47IAr0ZqCoPv5PJmu2lvHhjGj06HeOyakd+MWe8dvsucM1rztf6dm1wRpfMuAiS0mDrN5D2E2f+8KYsBOF2OWPHF/0L8r53rso7xMGUDyHhON+dkzHGK0E6MXbLMvu7bF7LyOaOs/szdlCCf19s5euwbRGMe6ThMAeIS4WbP3Gu4LctggmPw4V/b/qqPiGhcN4f4eKnITfDCfFbPrEwNyZA7Ardz5Zt283D81dx5oB47hk/wL8vVlkKHz4EPUfA8GuP3LZDV7j+HSjb6QwPPBYjb4TUcyEq3pZ6MyaALND9qGBPFbe9vJQendrz9KTh/r2ByO2Gj37rBPSk2c7dlo0JDTv2MN+nY0/fPI8x5qhZoPvR3z9az+7yGubdPorOHRqYOdBXaipg3m3Oqj0n3wFJI/33WsaYFssC3U9yiyt4Y2k2k05KZkjPjt4/sKayaYs07NkBc66B3GUw7vcH5gg3xrQ59qGon/zn840A3HpWP+8ftHQm/DkJ5t/pzEDYmNpqeOlSyF8DV78Mp/3cxnwb04ZZoPvB9pIK5qbncGVaLxI7ezFE0e1yPsx89y5nnu9lL8HLlzszER7JkmegYA1c8SIMvsg3xRtjWi3rcvGDZz7fhFuV28704uq8tgpev8m52WfUVOfuylVvOFfpz4+D4y5z2kmI8/2+1exLt8MXj8OACTDwfP+djDGm1bBA97GdpZXMTs/mipFJ9OraofEHLPqXE+YTHj8ws+AJk6BLCrwx5cDkVuqGRdPgR8/DoAsOLMU24c9+OxdjTOtige5D++4IdbuVn53Vv/EHlOTAl3915viuP01s8hi4N/PA9p4dzhwpc66BEdfDyrlwxn22FJsxZj+v+tBFZIKIrBORjSLywBHaXSEiKiINrqYR7J75IosPMnfwqwmDSI714ur8w4ecK+9z/9h425jucNMCZ7WeZTOdpdhOs6XYjDEHNHqFLiKhwDRgPJADpIvIfFVdXa9dDM56okv8UWhL99WGAp5cuJaLhvXg5tP7NP6ArC+cceNn/cb7+cDbtYcfvQh9z3LmEg/34oeGMabN8KbLZRSwUVWzAERkDjARWF2v3WPAE8AvfVphK5BdtJc7Zy8ntVsMT1wxrOFFK9wu2PQpVJc525//xZnG9tS7mvZiISHOrfbGGFOPN4GeCGTX2c4BRtdtICInAr1U9T0ROWygi8hUYCpAcnJy06ttoR6en4nLpTx73Ug6hB/mr/TzPzvzje8jITDpVeeq2xhjfMCbQG/oThXdf1AkBHgKuLGxJ1LV6cB0gLS0NG2keauwdOtuPl2bz/0TBpISF9Vwo8JN8M3TzrDDM3/l7Ivo6Lt5VIwxBu8CPQfoVWc7Ccirsx0DDAU+93Q1dAfmi8glqprhq0JbFFVY8iysfY/I7aW83t7FiM2dYYvnZ1+nJDjvT86MhgALf+MshjzhL86Hm8YY4wfeBHo6kCoifYBcYBJwzb6DqloCxO3bFpHPgV8GbZjXVsN798D3L1PeeSBlFULvrh0IVbfn9xaFVW/CtsVwzVwoyoL1H8C5f7AwN8b4VaOBrqq1InIHsBAIBV5U1UwReRTIUNX5/i6yxSgvhLnXwdZv0DPu58a1Z5IdWcXnt58F7UIPtNu2xBkv/vw4iIiGuIEw+tbDPq0xxviCVzcWqeoCYEG9fb87TNuzjr2sFuq9uyEnAy5/ni8iziT9w3T+cOlQIuuGOUDyaLjlU3j1ameulevm2cIPxhi/sztFveV2OWPHh18Dw67kn//+hqQu7bkqrVfD7bv0hps/hoJ1Nj+5MaZZ2GyL3tqZCVWl0PsUMrYUsWxbMbec3pfwsCP8FUZEW5gbY5qNBbq3ti1yviaP4dkvs+jcoR1XpiUFtiZjjKnDAt1bW7+FjklsqunKx2t2cv2Y3oe/icgYYwLAAt0bqs4wxN4n8/xXmwkPDeH6U1ICXZUxxhzEAt0buzdD2Q72JJzEm8ty+NHIJOKiIwJdlTHGHETDguYAAA2wSURBVMQC3Rtbnf7zNwp6UeNyc/NpXsymaIwxzcwC3RvbvsUd0Zknl8H5Q7vTNz460BUZY8whLNC9sXURq8KG4FLh1+cPDnQ1xhjTIAv0xpTlQ9Em3ivuzc/O6u/dOqHGGBMAFuiNqN3yLQBbo0/gp2fa+p3GmJbLAr0Ra5cspELDufKiiw6ds8UYY1oQC/QjcFVX0iX7Y7IiB3POULsr1BjTslmgH8G2958gkXz2nnRnw+uEGmNMC2KBfjglOfRcMY2PGMXxZ14e6GqMMaZRFuiHUfvBg6jbxbKBv7S+c2NMq2CB3pCsLwhbM49/107kzNFpga7GGGO84lWgi8gEEVknIhtF5IEGjt8qIitF5HsR+VpEhvi+1GaiCgsfJD+sO/OjfsSolK6BrsgYY7zSaKCLSCgwDTgfGAJMbiCwX1XV41V1OPAE8HefV9pcsr+DnSt5qvJizh/Rl5AQ+zDUGNM6eHOFPgrYqKpZqloNzAEm1m2gqqV1NqMA9V2JzWzZTGpCO/BO7clcdmJioKsxxhivebNCQyKQXWc7Bxhdv5GI3A7cC4QDYxt6IhGZCkwFSE5Obmqt/ldZAqve4tN2Z5LSoxsDEmICXZExxnjNmyv0hvocDrkCV9VpqtoP+BXwUENPpKrTVTVNVdPi4+ObVmlzWPk61Fbwr5LT7OrcGNPqeBPoOUDdpe2TgLwjtJ8DXHosRQWEKiydQX7UADLpwyXDewa6ImOMaRJvAj0dSBWRPiISDkwC5tdtICKpdTYvBDb4rsRmkrccdqxkVvVZnNo/noSOkYGuyBhjmqTRPnRVrRWRO4CFQCjwoqpmisijQIaqzgfuEJFxQA2wG7jBn0X7xbKZuEMjmbFnFI9MsO4WY0zr49Wy9aq6AFhQb9/v6nx/t4/ral4lubBiLt93GktNdTTnDe0e6IqMMabJ7E5RgI9+i7pdPLT7As4d0p3oCK9+zhljTItigb75K1j1JlkDb2F1RVcb3WKMabXadqC7auF/90PnZP6v6iJio8I5LTUu0FUZY8xRaduBnv485K+m+Izfs2BtMRef0JN2oW37r8QY03q13fSqKoPP/gT9zuG5/MHUuN3ccEpKoKsyxpij1nYDfdWbUFVCxSn38fKSbM4b0p0+cVGBrsoYY45a2w30ZTMhfjCz8xIoqahh6pl9A12RMcYck7YZ6DtWQu5SXCdezwvfbOGklC6MSO4S6KqMMeaYtM1AXzoTQiP4sN1Z5BZXcMvpdnVujGn92l6gV++FFXPRIROZtriQvvFRjBucEOiqjDHmmLW9QF/9DlSVsD7pMlblljLltD62KpExJii0vUBfNhNi+/PM5h5ER4Rx6XC7M9QYExzaTqC7XfDhQ7BtEXuPv573V+3gshMTibJ5W4wxQaJtpFlVGbx1C6xbAKOm8goTqK7dyLVjege6MmOM8Zngv0KvqYD/ToD1H8D5T+Ke8AQvp+dxUkoXBna3NUONMcEj+AM9J8MZd37J/8HoqXy9cRdbC/fa1bkxJugEf6AXrHW+9hsLwCtLttI1KpwJtoiFMSbIeBXoIjJBRNaJyEYReaCB4/eKyGoRWSEin4hIy7n8zV8NEZ0gpgeFZVV8vCafK9OSiAgLDXRlxhjjU40GuoiEAtOA84EhwGQRGVKv2XIgTVWHAW8AT/i60KOWvxa6DQYRFmUV4nIr5x1nV+fGmODjzRX6KGCjqmapajUwB5hYt4Gqfqaqez2bi4Ek35Z5lFShYA10GwTAt5sKiY4IY1hipwAXZowxvudNoCcC2XW2czz7DmcK8L+GDojIVBHJEJGMgoIC76s8WmU7oWI3dHN+oVi0qZDRfboSZotYGGOCkDfJ1tB98dpgQ5FrgTTgyYaOq+p0VU1T1bT4+Hjvqzxa+Wucr/GDyCuuYPOuck7uF+v/1zXGmADw5saiHKBXne0kIK9+IxEZBzwInKmqVb4p7xjtG+HSbQiL1hUCcEo/WzPUGBOcvLlCTwdSRaSPiIQDk4D5dRuIyInAs8Alqprv+zKPUv5q6BAL0fF8u6mQLh3aMchuJjLGBKlGA11Va4E7gIXAGmCuqmaKyKMicomn2ZNANPC6iHwvIvMP83TNK38txA9GVVm0aRcn94u1mRWNMUHLq7lcVHUBsKDevt/V+X6cj+s6dqpOl8uwq9lauJe8kkpus+4WY0wQC97hHqW5UFUK3Qbx7aZ9/ef2gagxJngFb6DnH/hA9NtNu0joGEHfuKjA1mSMMX4UxIG+GgCNH8SiTYWc0i8OEes/N8YEr+AN9IK1EN2dVUWhFJZX2/hzY0zQC95Az18N3QbxxtJswsNCOHeILQRtjAluwRnobjcUrKM2diDzvs/jvOO607lDeKCrMsYYvwrOQC/ZBjV7yaxNpKSihqvTejX+GGOMaeWCM9C3LQHg9bxYEju3t+GKxpg2ITgDff0HuDp049VtnbhiZJLdHWqMaROCL9BdNbDxE9bGnIxbQ7hiZMuYmt0YY/wt+AJ922KoKuGV3YM5tX8svbp2CHRFxhjTLIIv0Nd/gCukHfNKB3CVfRhqjGlDgi7QXes+YIn7OPolJnDh8T0CXY4xxjSb1hnoblfD+ws3EVq0kY9cw3nyymG21Jwxpk1pfYm35j14biyU5B5yaN1XbwCQPPoyBnXv2NyVGWNMQLW+QA+LhMJNTqjnLtu/e92OPZT88C5bQpL58YQzAligMcYEhlcLXLQoqeNgyocw+2r0vxew8oSHmLM1hrW5RbwWvpri4VMJD2t9P6eMMeZYeZV8IjJBRNaJyEYReaCB42eIyDIRqRWRK3xfZj0JQ9CbP2FreD+GLX2QP+26i7ciHqGduIgfeZnfX94YY1qiRq/QRSQUmAaMB3KAdBGZr6qr6zTbBtwI/NIfRTbkr98U8VzRfTw2vJgrhicQKgIRMZA8urlKMMaYFsWbLpdRwEZVzQIQkTnARGB/oKvqFs8xtx9qPMSLX29m2mebmDyqH1ddNtQWrjDGGLzrckkEsuts53j2NZmITBWRDBHJKCgoOJqn4J3vc3n0vdVMOK47f7jUwtwYY/bxJtAbSkw9mhdT1emqmqaqafHx8UfzFCR0jGT8kAT+MWk4oTbpljHG7OdNl0sOUPce+iQgzz/lNG5M31jG9LXpcI0xpj5vrtDTgVQR6SMi4cAkYL5/yzLGGNNUjQa6qtYCdwALgTXAXFXNFJFHReQSABE5SURygCuBZ0Uk059FG2OMOZRXNxap6gJgQb19v6vzfTpOV4wxxpgAsVsqjTEmSFigG2NMkLBAN8aYIGGBbowxQcIC3RhjgoSoHtVNn8f+wiIFwNajfHgcsMuH5bQWbfG82+I5Q9s877Z4ztD08+6tqg3eah+wQD8WIpKhqmmBrqO5tcXzbovnDG3zvNviOYNvz9u6XIwxJkhYoBtjTJBorYE+PdAFBEhbPO+2eM7QNs+7LZ4z+PC8W2UfujHGmEO11it0Y4wx9VigG2NMkGh1gS4iE0RknYhsFJEHAl2PP4hILxH5TETWiEimiNzt2d9VRD4SkQ2er10CXauviUioiCwXkfc8231EZInnnF/zzMkfVESks4i8ISJrPe/5yW3kvb7H8+97lYjMFpHIYHu/ReRFEckXkVV19jX43orjn55sWyEiI5r6eq0q0EUkFJgGnA8MASaLyJDAVuUXtcAvVHUwMAa43XOeDwCfqGoq8IlnO9jcjTPv/j6PA095znk3MCUgVfnX08AHqjoIOAHn/IP6vRaRROAuIE1VhwKhOIvnBNv7PQOYUG/f4d7b84FUz5+pwH+a+mKtKtCBUcBGVc1S1WpgDjAxwDX5nKpuV9Vlnu/34PwHT8Q515meZjOBSwNToX+ISBJwIfC8Z1uAscAbnibBeM4dgTOAFwBUtVpViwny99ojDGgvImFAB2A7QfZ+q+qXQFG93Yd7bycCL6ljMdBZRHo05fVaW6AnAtl1tnM8+4KWiKQAJwJLgARV3Q5O6APdAleZX/wDuB9we7ZjgWLPqlkQnO93X6AA+K+nq+l5EYkiyN9rVc0F/gpswwnyEmApwf9+w+Hf22POt9YW6NLAvqAddyki0cCbwM9VtTTQ9fiTiFwE5Kvq0rq7G2gabO93GDAC+I+qngiUE2TdKw3x9BtPBPoAPYEonC6H+oLt/T6SY/733toCPQfoVWc7CcgLUC1+JSLtcML8FVV9y7N7575fwTxf8wNVnx+cClwiIltwutLG4lyxd/b8Sg7B+X7nADmqusSz/QZOwAfzew0wDtisqgWqWgO8BZxC8L/fcPj39pjzrbUFejqQ6vkkPBznQ5T5Aa7J5zx9xy8Aa1T173UOzQdu8Hx/A/BOc9fmL6r6a1VNUtUUnPf1U1X9MfAZcIWnWVCdM4Cq7gCyRWSgZ9c5wGqC+L322AaMEZEOnn/v+847qN9vj8O9t/OB6z2jXcYAJfu6Zrymqq3qD3ABsB7YBDwY6Hr8dI6n4fyqtQL43vPnApw+5U+ADZ6vXQNdq5/O/yzgPc/3fYHvgI3A60BEoOvzw/kOBzI87/c8oEtbeK+B3wNrgVXALCAi2N5vYDbOZwQ1OFfgUw733uJ0uUzzZNtKnBFATXo9u/XfGGOCRGvrcjHGGHMYFujGGBMkLNCNMSZIWKAbY0yQsEA3xpggYYFujDFBwgLdGGOCxP8DF/V0OyDl4jUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores1)\n",
    "plt.plot(test_scores1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  ReLU + Sigmoid + Cross Entropy + L2 + Momentum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "net2 = Net()\n",
    "criterion2 = nn.CrossEntropyLoss()\n",
    "optimizer2 = optim.SGD(net2.parameters(), lr = 1e-1, weight_decay=1e-2, momentum=0.5)\n",
    "loss_scores2, train_scores2, test_scores2 = fit(net2, criterion2, optimizer2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXhV5bn+8e+TAQJJSAIJEELCPMkMERBnqwLOrdaKY+tAHas9trX2/HraY09bT9vjUGtbUalDW/SoValacRapgCQyj4YxEyQQCPOQ5Pn9sbeeSBMIkGTt7Nyf69pXst61VvazrgV3Vt717vWauyMiItErJugCRESkaSnoRUSinIJeRCTKKehFRKKcgl5EJMrFBV1AXdLT071nz55BlyEi0mLk5+dvcfeMutZFZND37NmTvLy8oMsQEWkxzGxDfevUdSMiEuUU9CIiUU5BLyIS5RT0IiJRTkEvIhLlFPQiIlFOQS8iEuUichz9sXr4nc/omtKWwd1S6NclibZxsUGXJCISuKgJ+gNVNTw5ey079lUBEB9rDOueyhn9Mzh9QAZDuqUQE2MBVyki0vwsEiceyc3N9WP5ZGxNjbOhYg/LSipZUlzJnDVbWVxUCUDn5LZMGNyViUO6MrZXR+Ji1WslItHDzPLdPbfOdUcKejPLBp4BugI1wFR3f/iQba4C7gkv7gJucfdF4XXrgZ1ANVBVXyG1HWvQ12XLrv3MWl3OW8s288HqMvYdrKFjYhsmDO7KhcMyGaPQF5EocLxBnwlkuvunZpYM5AOXuPvyWtuMB1a4+zYzmwT81N3HhtetB3LdfUtDC27MoK9t74FqPlxdxutLNvHuis3sOVBNelIbzhuayYXDuzE6J03dOyLSIh0u6I/YR+/upUBp+PudZrYCyAKW19rm41q7zAW6H1fFTaRdm1gmDslk4pBM9h6o5v1VZby+uJTn5xfyzJwNZKYkcMGwUOgPzUrBTKEvIi3fUfXRm1lPYBYwxN131LPN94CB7n5jeHkdsA1w4DF3n3qk92mqK/r67NpfxbsrNvP3RSV8uLqcg9VOj07tuXh4Ny4a0Y2+nZObrRYRkWNxXF03tX5IEvAh8HN3/1s925wJ/B44xd23htu6uXuJmXUG3gbucPdZdew7BZgCkJOTM3rDhnqfuNmkKvcc5M1lpfx9USkfr9lCjcMJmR24eEQo9DNT2gVSl4jI4Rx30JtZPPAaMNPdH6hnm2HAy8Akd19dzzY/BXa5+28O937NfUVfn7Kd+3h9cSmvLCxhUeF2zGBsr458dWQWE4dkktIuPugSRUSA478Za8DTQIW731XPNjnAe8C1tfvrzSwRiAn37ScSuqK/z93fPNx7RkrQ17Z+y25mLCrhlQXFrN2ymzZxMZw9qDOXjurOaf0ziNfIHREJ0PEG/SnAR8ASQsMrAX4E5AC4+x/N7AngUuDz/pYqd881s96ErvIhdOP3r+7+8yMVHIlB/zl3Z3FRJS8vKGbGohIqdh8gPakNF4/I4uu53RnYtUPQJYpIK9QoffTNKZKDvraD1TV8sKqcl/KLeHflZg5WO8O6p/D13GwuGt5NXTsi0mwU9M2gYvcBXl1YzPPzC1m5aScJ8TFcMKwbk8fkMConVUM1RaRJKeibkbuzpLiS6Z8UMmNhMbsPVDOwazJXj+vBJSOzSGobNY8XEpEIoqAPyO79VcxYVMKzczawvHQHSW3juHRUFteN70nvjKSgyxORKKKgD5i7s6BwO3+es4HXFpdyoLqGMwZkcMMpvTilb7q6dUTkuCnoI0j5zv38dd5G/jxvA+U79zOwazI3ndqbC4d3o02chmiKyLFR0Eeg/VXVzFhYwhMfrWPV5p1kpiRw46m9mTwmm/Zt1I8vIkdHQR/B3J0PV5fzhw/WMG9dBWnt47nhlF5cN74nyQkanikiDaOgbyHyN1Tw6PtreG9lGSnt4rnpVAW+iDSMgr6FWVJUyUPvrObdlWWktY/ntjP7cvW4HiTEaw5cEambgr6FWlS4nd+8tYqPPttCZkoC3z2nP5eO6k6sJkcRkUMcLug1zCOCDc9O5dkbxvLXG8fSuUMCP3hxMRc8MpuPCxo8WZeIiIK+JRjfN51Xbh3PI5NHsmPvQa58Yh43Pp3Hxq17gi5NRFoABX0LYWZcOLwb7959Oj+YOICP12zh7Ac/5KF3VrPvYHXQ5YlIBFPQtzAJ8bHcekZf3r37dM49oQsPvfMZ5z44i9mfqTtHROqmoG+hMlPa8bsrR/HXG8cSF2Nc/eQ8vvfCIrbtPhB0aSISYRT0Ldz4vum8ceep3H5mX15ZUMw5D37IW8s2BV2WiEQQBX0USIiP5XsTBvD3O06hS4cEpjybz/deWMSOfQeDLk1EIoCCPooMyuzAy7eezB1n9eXlBcVMeugj8tZXBF2WiATsiEFvZtlm9r6ZrTCzZWZ2Zx3bmJn91swKzGyxmY2qte46M/ss/LqusQ9AvqxNXAx3nzuAF28+ibhY4xtT5/Lo+wXU1ETeB+NEpHk05Iq+Crjb3QcB44DbzOyEQ7aZBPQLv6YAfwAws47AT4CxwBjgJ2aW1ki1y2GMzEnjtTtO4byhmfx65iqunfYJW3btD7osEQnAEYPe3Uvd/dPw9zuBFUDWIZtdDDzjIXOBVDPLBCYAb7t7hbtvA94GJjbqEUi9khPi+e0VI7j/a0OZv76Cix6ZzeKi7UGXJSLN7Kj66M2sJzASmHfIqiygsNZyUbitvva6fvYUM8szs7zy8vKjKUsOw8y4YkwOL90yHjPjsj/O4YW8wiPvKCJRo8FBb2ZJwEvAXe6+49DVdezih2n/10b3qe6e6+65GRkZDS1LGmhIVgozbj+Z3B5pfP/Fxfz89eXqtxdpJRoU9GYWTyjk/+Luf6tjkyIgu9Zyd6DkMO0SgE5JbXnm+jFcM64Hj3+0jlv/8il7D+jxCSLRriGjbgx4Eljh7g/Us9kM4Nrw6JtxQKW7lwIzgXPNLC18E/bccJsEJC42hvsuHsyPLziBmcs3MfnxubpJKxLlGnJFfzJwDXCWmS0Mv84zs5vN7ObwNm8Aa4EC4HHgVgB3rwB+BswPv+4Lt0mAzIwbTunFY1ePZuWmHVz+xzkUb98bdFki0kQ08Ugrl7e+gm89NZ/ktnH8+cax9M5ICrokETkGmnhE6pXbsyPPTRnH/qoaLn9sDstLDr3PLiItnYJeGNwthRduPon42BiufnIeqzbtDLokEWlECnoBoHdGEtNvGkd8rHHl43P5bLPCXiRaKOjlCz3TE5l+0zhiYozJj89jTfmuoEsSkUagoJcvCV3ZjwWca5/8hE2V+4IuSUSOk4Je/kXfzsk89a0xbN9zgOumfULlXj3XXqQlU9BLnYZkpfDYNbms3bKLm57J0wTkIi2Ygl7qdUq/dP7n8hF8sq6Cu19YRCR+5kJEjiwu6AIksl00vBsl2/dy/z9WMqBLMt/5Sr+gSxKRo6SglyP69mm9Wb1pJw+8vZr+XZKYOCQz6JJE5Cio60aOyMz4xdeGMjInle8+v4hlJZVBlyQiR0FBLw2SEB/LY9eMJrV9PN9+Np/KPRqJI9JSKOilwTonJ/DoVaPYVLmP772om7MiLYWCXo7KqJw07j1vEG8v38yTs9cFXY6INICCXo7a9Sf3ZMLgLtz/j5Xkb9D0AiKRTkEvR83M+NVlw+mW2o7vTF/Ijn3qrxeJZAp6OSYp7eJ5+IoRbNqxj5++uizockTkMBT0csxG5qRx+5l9+duCYl5fXBp0OSJSj4ZMDj7NzMrMbGk9679fay7ZpWZWbWYdw+vWm9mS8DrNDRiFbj+rL8O7p/Cjl5foSZciEaohV/RPARPrW+nuv3b3Ee4+ArgX+PCQCcDPDK+vcy5DadniY2N48Bsj2F9VzQ9eWqwhlyIR6IhB7+6zgIYOrZgMTD+uiqTF6Z2RxL2TBjFrdTkvfVocdDkicohG66M3s/aErvxfqtXswFtmlm9mU46w/xQzyzOzvPLy8sYqS5rJNeN6kNsjjZ+9tpzynfuDLkdEamnMm7EXAv88pNvmZHcfBUwCbjOz0+rb2d2nunuuu+dmZGQ0YlnSHGJijPsvHcbeA9X89O8ahSMSSRoz6K/gkG4bdy8Jfy0DXgbGNOL7SYTp2zmJ73ylL68vLuWtZZuCLkdEwhol6M0sBTgdeLVWW6KZJX/+PXAuUOfIHYke3z69DwO7JvPjV5eya39V0OWICA0bXjkdmAMMMLMiM7vBzG42s5trbfZV4C13312rrQsw28wWAZ8Ar7v7m41ZvESe+NgYfvm1oWzesZ+H31kddDkiQgMmHnH3yQ3Y5ilCwzBrt60Fhh9rYdJyjcxJ44oTs5n2z/VcNjqbAV2Tgy5JpFXTJ2OlSfxg4kCSE+L48atLNbZeJGAKemkSHRPb8IMJA/lkXQWvLiwJuhyRVk1BL03mihOzGZ6dyn+9voKdesKlSGAU9NJkYmKM+y4azJZd+/n9B2uCLkek1VLQS5Manp3K10Zl8eRH6yis2BN0OSKtkoJemtwPJgwkNsb45T9WBF2KSKukoJcm1zUlgVvO6MMbSzYxb+3WoMsRaXUU9NIsbjq1N91SErjvteXU1Gi4pUhzUtBLs2jXJpZ7Jg1kWckOXlmoRxmLNCcFvTSbC4d1Y1j3FH4zcxX7DlYHXY5Iq6Ggl2YTE2PcO2kQJZX7eOrj9UGXI9JqKOilWZ3UpxNfGdiZR98vYNvuA0GXI9IqKOil2d0zaSC791fxyHsFQZci0ioo6KXZ9e+SzOW52Tw7dz0bt+pDVCJNTUEvgfjuOf2JjTEeeHtV0KWIRD0FvQSiS4cEvnVyL15dVMLykh1BlyMS1RT0EpibT+9Dh4R4fjVzZdCliES1hkwlOM3MysyszvlezewMM6s0s4Xh13/UWjfRzFaZWYGZ/bAxC5eWL6VdPLee0YcPVpUzZ40ejSDSVBpyRf8UMPEI23zk7iPCr/sAzCwWeBSYBJwATDazE46nWIk+143vSWZKAve/uVIzUYk0kSMGvbvPAiqO4WePAQrcfa27HwCeAy4+hp8jUSwhPpa7zu7HosLtzFy2KehyRKJSY/XRn2Rmi8zsH2Y2ONyWBRTW2qYo3FYnM5tiZnlmlldeXt5IZUlLcOmo7vTJSOTXM1dRVV0TdDkiUacxgv5ToIe7DwceAV4Jt1sd29b7t7m7T3X3XHfPzcjIaISypKWIi43h+xMGsKZ8N3/7VA88E2lsxx307r7D3XeFv38DiDezdEJX8Nm1Nu0OaJZoqdOEwV0Znp3Kg++s1gPPRBrZcQe9mXU1Mwt/Pyb8M7cC84F+ZtbLzNoAVwAzjvf9JDqZGfdMHEBp5T7+PHdD0OWIRJW4I21gZtOBM4B0MysCfgLEA7j7H4HLgFvMrArYC1zhoeETVWZ2OzATiAWmufuyJjkKiQrj+6Rzar90fvd+AZefmE2HhPigSxKJChaJQ9pyc3M9Ly8v6DIkAEuLK7ngkdncfmZfvjdhQNDliLQYZpbv7rl1rdMnYyWiDMlK4cLh3Xhy9jrKduwLuhyRqKCgl4hz9zn9OVhdw2/f+yzoUkSigoJeIk7P9ESuHJvD9E8KWbdld9DliLR4CnqJSHec1Y+2cTH85i09xljkeCnoJSJlJLflxlN78/riUhYXbQ+6HJEWTUEvEeumU3vRKbENv3hjhR54JnIcFPQSsZIT4rnz7H7MXVvB+6vKgi5HpMVS0EtEmzwmh17pifzyjZV64JnIMVLQS0SLj43hnokD+KxsFy/kFwVdjkiLpKCXiDdhcFdye6TxwNur2b2/KuhyRFocBb1EPDPj3vMGUb5zP1NnrQ26HJEWR0EvLcLoHmmcPyyTx2atobRyb9DliLQoCnppMX44cSA1Dr96Ux+iEjkaCnppMbI7tuemU3vx8oJiFmzcFnQ5Ii2Ggl5alFvO6EtGclt+9tpyfYhKpIEU9NKiJLWN4/vnDuDTjduZsUgzU4o0hIJeWpxLR3dnSFYHfvHGCg23FGkABb20OLExxn9eNITNO/bzyHsFQZcjEvGOGPRmNs3MysxsaT3rrzKzxeHXx2Y2vNa69Wa2xMwWmpnmBpRGM7pHGpeN7s6Ts9eypnxX0OWIRLSGXNE/BUw8zPp1wOnuPgz4GTD1kPVnuvuI+uYyFDlW90wcSEJ8LD+dsUw3ZkUO44hB7+6zgIrDrP/Y3T8f6zYX6N5ItYkcVkZyW/7tnP589NkWZi7bFHQ5IhGrsfvobwD+UWvZgbfMLN/MphxuRzObYmZ5ZpZXXl7eyGVJtLpmXA8Gdk3mP/++nF26MStSp0YLejM7k1DQ31Or+WR3HwVMAm4zs9Pq29/dp7p7rrvnZmRkNFZZEuXiYmP4+VeHsmnHPh54a3XQ5YhEpEYJejMbBjwBXOzuWz9vd/eS8Ncy4GVgTGO8n0hto3ukcdXYHJ76eB1LiiqDLkck4hx30JtZDvA34Bp3X12rPdHMkj//HjgXqHPkjsjx+v6EgXRKasu9Ly/WBCUih2jI8MrpwBxggJkVmdkNZnazmd0c3uQ/gE7A7w8ZRtkFmG1mi4BPgNfd/c0mOAYRUtrF85MLT2Bp8Q6enrMh6HJEIopF4rC03Nxcz8vTsHs5Ou7O9U/NZ+7aCt6861R6dEoMuiSRZmNm+fUNY9cnYyVqmBm/+NpQ4mKMe15aTE1N5F3EiARBQS9RJTOlHf9+/iDmrq3gL59sDLockYigoJeo840Tszm1Xzr3v7GCwoo9QZcjEjgFvUQdM+P+S4cBqAtHBAW9RKms1Hb8+IIT+HjNVqb9c13Q5YgESkEvUesbJ2Zzzgld+NWbq1i5aUfQ5YgERkEvUcvMuP9rQ+nQLp67nlvIvoPVQZckEggFvUS1Tklt+fVlw1i5aSe/nrkq6HJEAqGgl6h35sDOXHtSD56cvY53V2wOuhyRZqegl1bhR+cN4oTMDtz9wiJKtu8NuhyRZqWgl1YhIT6WR68axcGqGu6YvoCDevCZtCIKemk1eqUn8stLh5G/YRu/UX+9tCIKemlVLhrejavG5vDYrLX8Y0lp0OWINAsFvbQ6/3HhCYzITuXuFxaxevPOoMsRaXIKeml12sbF8serR9O+TRzffjafyr0Hgy5JpEkp6KVV6pqSwB+uHkVhxR6++/xCqvU8HIliCnpptU7s2ZGfXDSY91aW8cs3VgRdjkiTiQu6AJEgXTOuB2vKdvHE7HX0ykjkqrE9gi5JpNE16IrezKaZWZmZ1Tm5t4X81swKzGyxmY2qte46M/ss/LqusQoXaSz/7/xBnDkgg/94dRkffVYedDkija6hXTdPARMPs34S0C/8mgL8AcDMOgI/AcYCY4CfmFnasRYr0hTiYmN45MpR9OucxK1//pTlJXrSpUSXBgW9u88CKg6zycXAMx4yF0g1s0xgAvC2u1e4+zbgbQ7/C0MkEElt45j2zRNJSojjuj99opmpJKo01s3YLKCw1nJRuK2+9n9hZlPMLM/M8srL9eezNL9uqe145voxHKiq4dppn7B11/6gSxJpFI0V9FZHmx+m/V8b3ae6e66752ZkZDRSWSJHp1+XZKZ980RKK/fyrafms3OfxthLy9dYQV8EZNda7g6UHKZdJGKN7pHG768axfKSHVz/1Hx2768KuiSR49JYQT8DuDY8+mYcUOnupcBM4FwzSwvfhD033CYS0c4a2IWHrxhJ/oZt3Ph0nmankhatQePozWw6cAaQbmZFhEbSxAO4+x+BN4DzgAJgD/Ct8LoKM/sZMD/8o+5z98Pd1BWJGOcPy+RA9XD+7X8XMeXZfKZeM5qE+NigyxI5auYeeR/9zs3N9by8vKDLEAHg+fkbueelJZzaL52p1+TSro3CXiKPmeW7e25d6/QIBJEj+MaJOfzqsmHMLtjCN//0CbvUZy8tjIJepAEuz83moW+MIG/DNq59cp6eeCktioJepIEuHpHFo1eOZElxJd94bA6bd+wLuiSRBlHQixyFiUMy+dM3x1BYsYev/f5j1pTvCrokkSNS0IscpVP6pfP8t09if1U1l/3hY/I3aCCZRDYFvcgxGJKVwku3jCelXTyTH5/HqwuLgy5JpF4KepFj1KNTIi/fejIjslO587mFPPTOaiJxuLKIgl7kOKQltuHZG8Zw6ajuPPTOZ9z+1wV6ZIJEHM0wJXKc2sbF8puvD6NflyR+9eZKCsp28dg1o+mZnhh0aSKAruhFGoWZcfPpfXj6+jFs3rmPi343m3dXbA66LBFAQS/SqE7tl8Hfbz+F7I7tueHpPH7xxgoOVtcEXZa0cgp6kUaW3bE9L90ynmvG9WDqrLVc/tgcirZpxioJjoJepAkkxMfys0uG8OiVoyjYvItJD33EywuKNCpHAqGgF2lC5w/L5I07T2VA12S++/wi7pi+gMo9ek6ONC8FvUgTy+7Ynue/fRLfnzCAN5du4pwHP+Sd5bpRK81HQS/SDGJjjNvO7Msrt51Mx8Q23PhMHnc+t4Btuw8EXZq0Agp6kWY0JCuFGbefwl1n9+P1xaV85YEPeSlffffStBoU9GY20cxWmVmBmf2wjvUPmtnC8Gu1mW2vta661roZjVm8SEvUJi6Gu87uz2vfOYUendpz9wuLuPLxeXoSpjSZI04laGaxwGrgHKCI0Pyvk919eT3b3wGMdPfrw8u73D3paIrSVILSWtTUONPnb+S//7GSvQeruf7kXtx+Vl+SE+KDLk1amOOdSnAMUODua939APAccPFhtp8MTD/6MkVan5gY46qxPXj37jO4ZEQWj81ay1n/8yEv5hdRU6PuHGkcDQn6LKCw1nJRuO1fmFkPoBfwXq3mBDPLM7O5ZnZJfW9iZlPC2+WVl5c3oCyR6JGR3JZff304r9x2Mt1S2/G9FxZxwSOz+WfBlqBLkyjQkKC3Otrqu9S4AnjR3atrteWE/5y4EnjIzPrUtaO7T3X3XHfPzcjIaEBZItFnRHYqL98ynoevGEHl3oNc9cQ8vvmnT1haXBl0adKCNSToi4DsWsvdgZJ6tr2CQ7pt3L0k/HUt8AEw8qirFGlFYmKMi0dk8e7dp/Oj8wayYON2LnhkNrf95VMKynTDVo5eQ4J+PtDPzHqZWRtCYf4vo2fMbACQBsyp1ZZmZm3D36cDJwN13sQVkS9LiI9lyml9+OieM/nOWX15f1UZ5z74IXc+t4CCsp1BlyctyBGfR+/uVWZ2OzATiAWmufsyM7sPyHP3z0N/MvCcf3kYzyDgMTOrIfRL5f76RuuISN06JMTzb+cO4NrxPXl81lqembOBGYtKOH9oJree0ZcTunUIukSJcEccXhkEDa8Uqd/WXft5YvY6nvl4PbsPVHPGgAxuPr0PY3t1xKyuW2rSGhxueKWCXqSFqtxzkD/P28C02evYuvsAQ7NSuPHUXpw3NJP4WH3ovbVR0ItEsX0Hq3kxv4hp/1zH2vLddO2QwNXjcrhiTA7pSW2DLk+aiYJepBWoqXE+WF3GtNnrmV2whTaxMZw/LJOrxuYwukeaunWi3OGCXpODi0SJmBjjrIFdOGtgFwrKdvHnuRt4Mb+IlxcU079LEpPH5PDVkVmktm8TdKnSzHRFLxLFdu+v4rXFJfx13kYWFVXSJjaGcwd34fLcbE7um05sjK7yo4W6bkSEZSWVvJBXxCsLi9m+5yBdOrTlkhFZfHVUFgO7aohmS6egF5Ev7K+q5p3lZby8oIgPVpVTVeMM7JrMhcO7cdHwbmR3bB90iXIMFPQiUqetu/bz2uJSXl1YzKcbQ9NIDM9O5fyhXZk0JFOh34Io6EXkiAor9vDa4lLeWFLKkvBD1IZmpTBhcBcmDulK387JAVcoh6OgF5GjsnHrHt5YWsrMZZtYEL7S752eyFcGdebsQV0Y3SONOH0oK6Io6EXkmG3esY+3lm3i7RVlzFmzhYPVTkq7eE7rn8FZAzM4rV8GnfTBrMAp6EWkUezcd5CPPtvCeyvL+GBVGVt2HcAs1MVzWr8MTuufwcicVD2CIQAKehFpdDU1zpLiSj5cXc6s1eUsKNxOdY2T2CaWcb07cXLfdMb37UT/zsnEaLx+k1PQi0iTq9x7kDlrtjC7YAv/LNjKui27AeiU2IZxvTsxrndHxvXuRN/OSXocQxPQIxBEpMmltItn4pBMJg7JBKB4+17mrNnKx2u2MGfNVl5fUgqEgj+3Zxon9uzImF4dOSGzg27sNjFd0YtIk3N3Civ2MnftVuau28r89RUUVuwFoF18LCOyU8ntmcaonDRG5qTqeTzHQFf0IhIoMyOnU3tyOrXn8hNDU1CXVu5l/vpt5K+vIH/jNh59v4Ca8HVn74xERmSnMjI7leHZqQzs2oE2cbrqP1a6oheRiLB7fxWLirazYON2Pt2wjUVF29my6wAAbeJiGJTZgeHdUxialcKw7qn0yUhUl08tx30z1swmAg8TmjP2CXe//5D13wR+DRSHm37n7k+E110H/L9w+3+5+9NHej8FvYi4O8Xb97KwcDuLiypZVLidpcWV7D5QDUBCfCj8B3frwOBuKQzu1oH+XZJJiI8NuPJgHFfQm1kssBo4BygC5gOTa0/yHQ76XHe//ZB9OwJ5QC7gQD4w2t23He49FfQiUpfqGmfdlt0sLa5kSXElS4srWV6yg537qwCIjTH6ZCQyKLMDgzI7MLBrMoMyO9A5uW3Uj/Q53j76MUCBu68N/7DngIuB5YfdK2QC8La7V4T3fRuYCExvSOEiIrXFxhh9OyfRt3MSl4zMAkLj+TdW7GF56Q5WlO5gWckO5q+r4NWFJV/sl9Y+nv5dkhnYNZn+XZMZ0CWZfl2SSWkXH9ShNKuGBH0WUFhruQgYW8d2l5rZaYSu/r/r7oX17JtV15uY2RRgCkBOTk4DyhIRCc2s1TM9kZ7piZw3NPOL9u17DrCidCerNu1g1eadrNy0kxfzi77o+gHonNyW/l2Sv/jl8fmrU2KbqPoLoCFBX9fRHtrf83dgurvvN7ObgaeBsxq4b6jRfSowFUJdNw2oS0SkXqnt23BSn06c1KfTF23uTknlPlZv2snqzTtZvXkXBWU7+d+8QvbU+gWQ0i6ePhmJ9M5IondGIr3TQ197dISTebQAAAXzSURBVGpP27iWdw+gIUFfBGTXWu4OlNTewN231lp8HPjvWvuecci+HxxtkSIijcHMyEptR1ZqO84c2PmL9poaZ9OOfRSU7aKgbBdrykOvD1eX82J+0RfbxRh0S21Hr/REeqUn0rNTIj3T29OzUyLd09pH7BDQhgT9fKCfmfUiNKrmCuDK2huYWaa7l4YXLwJWhL+fCfzCzNLCy+cC9x531SIijSgmxuiW2o5uqe04rX/Gl9bt3HeQdVt2s7Z8N+u2/N/r5U+Lv7gJDP/3S6BHp/bkdAxd/ffo2J7s8CvI+wFHDHp3rzKz2wmFdiwwzd2Xmdl9QJ67zwC+Y2YXAVVABfDN8L4VZvYzQr8sAO77/MasiEhLkJwQz7DuqQzrnvqldnenYvcB1m/dzfote9hQsYf1W3azsWIPM5dtomL3gS9t3yEhLhT6ae3J7tiO7uGvWantyUprR1Lbpvv8qj4wJSLSBHbsO0hhxR4KK/awYeseCrftobBiL4Xb9lC8bS/7q2q+tH1q+3j6dU7ihZvHH9P76REIIiLNrENCfPiDXCn/ss7dKd+1n8KKvRRv30tROPyra5rmwltBLyLSzMyMzskJdE5OYHSPtCPvcJwi8xaxiIg0GgW9iEiUU9CLiEQ5Bb2ISJRT0IuIRDkFvYhIlFPQi4hEOQW9iEiUi8hHIJhZObDhGHdPB7Y0YjktQWs8Zmidx90ajxla53Ef7TH3cPeMulZEZNAfDzPLq+95D9GqNR4ztM7jbo3HDK3zuBvzmNV1IyIS5RT0IiJRLhqDfmrQBQSgNR4ztM7jbo3HDK3zuBvtmKOuj15ERL4sGq/oRUSkFgW9iEiUi5qgN7OJZrbKzArM7IdB19NUzCzbzN43sxVmtszM7gy3dzSzt83ss/DXpp/NoJmZWayZLTCz18LLvcxsXviYnzezNkHX2NjMLNXMXjSzleFzflK0n2sz+2743/ZSM5tuZgnReK7NbJqZlZnZ0lptdZ5bC/ltON8Wm9moo3mvqAh6M4sFHgUmAScAk83shGCrajJVwN3uPggYB9wWPtYfAu+6ez/g3fBytLkTWFFr+b+BB8PHvA24IZCqmtbDwJvuPhAYTuj4o/Zcm1kW8B0g192HALHAFUTnuX4KmHhIW33ndhLQL/yaAvzhaN4oKoIeGAMUuPtadz8APAdcHHBNTcLdS9390/D3Own9x88idLxPhzd7GrgkmAqbhpl1B84HnggvG3AW8GJ4k2g85g7AacCTAO5+wN23E+XnmtAUp+3MLA5oD5QShefa3WcBFYc013duLwae8ZC5QKqZZTb0vaIl6LOAwlrLReG2qGZmPYGRwDygi7uXQuiXAdA5uMqaxEPAD4Ca8HInYLu7V4WXo/Gc9wbKgT+Fu6yeMLNEovhcu3sx8BtgI6GArwTyif5z/bn6zu1xZVy0BL3V0RbV40bNLAl4CbjL3XcEXU9TMrMLgDJ3z6/dXMem0XbO44BRwB/cfSSwmyjqpqlLuE/6YqAX0A1IJNRtcahoO9dHclz/3qMl6IuA7FrL3YGSgGppcmYWTyjk/+Lufws3b/78T7nw17Kg6msCJwMXmdl6Qt1yZxG6wk8N/3kP0XnOi4Aid58XXn6RUPBH87k+G1jn7uXufhD4GzCe6D/Xn6vv3B5XxkVL0M8H+oXvzLchdPNmRsA1NYlw3/STwAp3f6DWqhnAdeHvrwNebe7amoq73+vu3d29J6Fz+567XwW8D1wW3iyqjhnA3TcBhWY2INz0FWA5UXyuCXXZjDOz9uF/658fc1Sf61rqO7czgGvDo2/GAZWfd/E0iLtHxQs4D1gNrAH+Peh6mvA4TyH0J9tiYGH4dR6hPut3gc/CXzsGXWsTHf8ZwGvh73sDnwAFwAtA26Dra4LjHQHkhc/3K0BatJ9r4D+BlcBS4FmgbTSea2A6ofsQBwldsd9Q37kl1HXzaDjflhAaldTg99IjEEREoly0dN2IiEg9FPQiIlFOQS8iEuUU9CIiUU5BLyIS5RT0IiJRTkEvIhLl/j/hibffYl0xuAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(loss_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3deXhU5dnH8e+dyQZZgJCFNZAAsqggGlkUFVFaLCp1q1DXloq2Yuu+tH19W23f7ra10gVcqnVBtGqjUFHrriAJCmICaFiTsCQkJJA9M3O/f8xAY0jIJExyMpP7c11zZc6ZZ87chxN+HJ55znNEVTHGGBP6IpwuwBhjTHBYoBtjTJiwQDfGmDBhgW6MMWHCAt0YY8JEpFMfnJycrMOHD3fq440xJiStXbt2n6qmtPSaY4E+fPhwcnNznfp4Y4wJSSKyo7XXrMvFGGPChAW6McaECQt0Y4wJExboxhgTJizQjTEmTFigG2NMmLBAN8aYMOHYOHRjjAkltQ0eyqrr2V/dyMG6xsPrG73K/uoGyqobqKxthACmJD9nbBoThvYNeo0BBbqIzAL+CLiAh1X1l81eHwY8CqQA5cCVqloU5FqNMabTqCpbSqvYUlpNeXUD5dUNFFfUUlBSxZaSKsqqGwLajkjbbVITY50JdBFxAYuAmUARkCMi2aqa36TZb4EnVPVxEZkB/AK4KujVGmNMOzS4vXi8R54xK8qeyjoKSqooKK1ifWEFOdv3U94stBNjIxmZGs85Y1MZ1j+O5Pho+vWOJrFXFIdy2xUh9IuLpn9cNImxUUREBJDonSSQM/RJQIGqbgUQkaXAHKBpoI8DbvE/fwt4KZhFGmPMIW6Pl8L9tewoqz4c1o0eZWd5NQUlVWwtrabkYD37qxs4WO8OaJtDk3oxY0wqk4YnMXZgIv3jo0mKiyY2ytWZuxJ0gQT6YKCwyXIRMLlZm/XAJfi6ZS4CEkSkv6qWNW0kIguABQDp6ekdrdkYE4ZUle1lNeRsK2fz3oOHu6LdXi/7axopr66n9GA92/fV0ODxtriN5PgYRqTEMTG9L0lxvrPp6MiWx34kxUUzKjWeEanxJMZGddZudalAAr2l/z80/z/M7cBDInIt8C5QDBzxT6OqLgYWA2RlZdnNTI3pKu56KPgPeBvbbhvsj/YqlTUNVNV7OFjfSHW9m4N1bt/PejdVdW6q6t2UHKznQK0vNqJdgsvfdSEi9ItxMTQmkoTYKNLGxpDWpxdpCTFEunxtIkToHxdNXIwbqA68uCr/o6sNGA9JGUHfbCCBXgQMbbI8BNjVtIGq7gIuBhCReOASVa0MVpHGmGP00nfhs3868tGRQH//o03Rrayv9z8OACXBqctRsx+ApPlB32wggZ4DjBKRDHxn3nOBbzZtICLJQLmqeoF78I14McZ0B1+84Qvz038A4y9v99sbPV7KaxrZvPsAG4oryd99gH1V9Ryoc+P192HHRLpIjI3kYL2bukYP4BvtkZkczwmD+5CZ3Js+vXxfJvbpFUWf3lHER7uQQIaEhKOEgZ2y2TYDXVXdIrIQWIlv2OKjqponIvcBuaqaDUwHfiEiiq/L5cZOqdYY0z4NNejyW/EmjWTPxFshMuaIJnWNHraVVvNFSRXb91VTVl1PWXXD4bHVB+v+23vaOzqGiemjGDE8jqTe0fTtHUWDx0t5VQPlNQ2MjI1iZGo8o1LjGTMwkT69wqNvOlSIBjAIvjNkZWWp3eDCmI6pqndTtL/m8Hjp2gbP4dcqahoPD8c7v3QJ3/K+wNyGH7PaO67N7SbHR5OSEEv/uOjDQ/GS/I8TBvfh+EGJRLnsAnMnichaVc1q6TW7UtSYEFBWVU/O9v3kbC9nzbZy8nZV0sLw6sOS46OZ3reUq73/YkPKbM4dfwkXxUYiLYxxiHQJw/rHMTI13s6oQ5wFujHd0K6KWnK2l/PRNl+AF5T4hmLEREYwMb0vC88eyegBiYfPnntH/3e8dEJsJH33rILnfgS9+3HitX/ixLiAvpI0Ic4C3Zgu5vEqFTW+rpJDfdX7qhvYVlpNQWkVX+w9yO7KOgDiYyLJGt6Pi08ezKThSZw4pA8xkUe52EUVVv8FXvsx9B8J854BC/MewwLdmI7wuOHN++GTJ0FbvsjFq9Dg8eL2ePGq73Jzr/ouoBGOHMonCBERgisCIhMiiIzwjcWWEnxD9T4KoC71Ql0FjDkfLvorxCQc866a0GGBbsxRqCq7K+vYVVF7+GxaasuZkXc3KSWrKB0yk8LGPv6LYhqJjowgNsqFx6uUVdej6jvLjo+NJDbKRUxUBLGRvp8xkS5io1zE+t/TK9rV4lV87ZY6Bk75NkTYl5c9jQW6MU0U7a8hZ3s5ez57j9jdayirbqDe/d8z8AiUK1xvkCj7ucO9gOcKphMZIZwwuA+jMxOoqndTVl2P26OckZXCeScOYFRqfM8db226lAW6CWtuj5eNuw+Ss72cz/ce9E3etK+aCBGS4qJIioum0aO+/uyqeg7UNXKdazl3Rz2DC/VNfNFs4IcnYTDFM1/g4rgTuBiYMLQPvaPtr5Jxnv0WmrBTerCe1/L38Hr+XnK376fKP+NeUlw0I1Pi+erxaQCHx3DHREZw/KBE0np5uWLvb8jc8yo6dg6c//sWL8RxRfUiPcKFTS9nuhsLdNP9eT3w1s9h1SLwtDy5lB56qNJP4XLgcvFN7CS9fCfa4gX2+h8tbsTftXLOvci0WwO7U4Ex3YgFuuneavfDP6+Dgtdh3Nd9Q/GAOreXPZW1FFfUUrS/lpKD9QD0j4tmREo8I1Lj6B8X0/4vGTOnQ8YZwdwDY7qMBbrpvko2wdJ5UFHo6/7I+jbV9W5+s3IzT6zajlchyiVMGNKXc6ek8dXjB5CRHOd01cY4xgLddE/52b4pX6N6w7WvQPoU3t5cwo9e/IxdlbXMm5TOhRMGcdLQviF3VxljOosFunGeKjRU/ff5hw/Cu79BB2exbcZfWP6F8OpL75G36wAjU+N5/oapnDIsydmajemGLNCNs6pK4Z/fhm3vfmn1qj5f486Sayhc8gUApwzrx73nj+OKKelHv/TdmB7MAt04Z9cnsPRKqNnHgcm38GFhI+sKK9jqSWFd9elMyuzP9Zn9mTkujbTEWKerNabbCyjQRWQWvhtAu4CHVfWXzV5PBx4H+vrb3K2qK4JcqwllqvDps1CUA4DX3QCfPkt1ZD9+l/wAT77XDxG4eOIQ7jgjg5F2daUx7dZmoIuIC1gEzMR3f9EcEclW1fwmzX4MLFPVv4jIOGAFMLwT6jWhqLEOlt8K655CY/tQ74mgttHDJ56x3FF9PX2iB3HNaanMn5bBoL69nK7WmJAVyBn6JKBAVbcCiMhSYA7QNNAVSPQ/70Ozm0ibHqyyGJ69EnZ9zPoR1/Od7edQWu1mUkYS10wdzsrMJJLjj7wa0xjTfoEE+mCgsMlyETC5WZufAK+JyE1AHHBuUKozoa2hBv4+G60u5dHB93N/3ghOG9GHRecex6QMG6ViTLAFEugtdWQ2v/nVPODvqvo7EZkK/ENETlD98kTRIrIAWACQnm4zYYS9d38N+7dxT8L/8ezW4dz+leP43vSRRERY37gxnSGQCZOLgKFNlodwZJfKfGAZgKquAmKB5OYbUtXFqpqlqlkpKSkdq9iEhr35eD/4Ey9xNssPjOSRa7JYOGOUhbkxnSiQQM8BRolIhohEA3OB7GZtdgLnAIjIWHyBXhrMQk3oqK1vZMfjC9jv7cWyfgvIvmkaM8akOV2WMWGvzUBXVTewEFgJbMQ3miVPRO4TkQv9zW4DrhOR9cAzwLWqepR7kptwVe/28PCDP2VYzQZWZd7M32+cZfOrGNNFxKnczcrK0tzcXEc+23Sefy/9M2dt/Ak1KRNIvvE1m4LWmCATkbWqmtXSa3bTQRMcXg+V2T/kvE33sLvXSJKvedLC3JguZoFujp0q+uyV9Pl4Ec/qTOKvfxUSrM/cmK5mgW6O3d7PkM0r+H3jJdTP+i1p/RLbfo8xJugs0M0xa8x7GS/CJ2kXc8XkYU6XY0yPZbMtmmN2cP2/2OIdxY3nT8Vl48yNcYydoZtj4i3fQdKBTWyIn2aX8xvjMAt0c0wK3l8GQPppl9p0t8Y4zALdHJPGvJfZKkM5a+pUp0sxpsezQDcdtmnrDkbXbaAifSZRLvtVMsZp9rfQdNjHbzxLpHg57sy5TpdijMEC3XRQYXkN/Yte50BUMvEZpzpdjjEGC3TTQQ+sWMc0WY9r7GyIsF8jY7oDG4du2m3tjnIiN75IXFQ9nHy50+UYY/zs1Mq0i9er3PdyPvOj38CbMhaGneZ0ScYYPwt00y7Z63chxWsZo1uJmPQdm1HRmG7EAt0ErK7Rw69e3cT3E95GoxNgvHW3GNOdBBToIjJLRDaLSIGI3N3C678XkXX+x+ciUhH8Uo3TXsvfS11lCdPd7yET5kJMgtMlGWOaaPNLURFxAYuAmfhuGJ0jItmqmn+ojare0qT9TcDETqjVOOxfnxQzP+4DIjyNcOp8p8sxxjQTyBn6JKBAVbeqagOwFJhzlPbz8N1X1ISR8uoG3vt8L1e63oBh0yB1rNMlGWOaCSTQBwOFTZaL/OuOICLDgAzgzVZeXyAiuSKSW1pa2t5ajYOWb9jNpfImfRt2w+TrnS7HGNOCQAK9pWEMrd1Zei7wvKp6WnpRVRerapaqZqWkpARao+kG3sndwA+jlqIZZ8LYC5wuxxjTgkACvQgY2mR5CLCrlbZzse6WsFNYXsMFexfRSxqQ2Q/YUEVjuqlAAj0HGCUiGSISjS+0s5s3EpHRQD9gVXBLNE775K1/Msf1IVWTfgDJo5wuxxjTijYDXVXdwEJgJbARWKaqeSJyn4hc2KTpPGCpqrbWHWNCkDbUkPXZzyh2DabPzDudLscYcxQBzeWiqiuAFc3W3dts+SfBK8t0FzvXvMIw3cObEx5icGSM0+UYY47CrhQ1R7Vj7UpqNZpTzr7I6VKMMW2wQDet2lNZR3JZDrsTx9MnId7pcowxbbBAN61a9t56xshOko6f4XQpxpgAWKCbFtU0uNmS+wYRovQde7bT5RhjAmCBblr0z7VFjHdvwOuKgcGnOF2OMSYAFujmCF6v8sj72zg7djOSPhlsdIsxIcEC3Rxh+Ybd7C8rIcO9DRl+htPlGGMCZIFuvqSmwc3/rdjIJf13IigMn+Z0ScaYAFmgmy9Z9FYBuyvrWDBsN0TGWv+5MSHEAt0ctm1fNUve3cbFJw9mQHkODDnV+s+NCSEW6AYAVeWnL+cRHRnBPWcPgD0bwPrPjQkpFugGgHe/2Mfbm0u5+dxRpOzLARSGn+50WcaYdrBANwA8+v420hJjuOa04bD53xDbB4ZOdrosY0w7WKAbdpRV887npcyblE4UXl+gj/oquKKcLs0Y0w4W6IanPtpJZIQwb1I6FK6G2nIYM9vpsowx7WSB3sPVNXpYllvIV48fQFpiLGxaDq4YGHmO06UZY9opoEAXkVkisllECkTk7lbafENE8kUkT0SeDm6ZprO8vH4XFTWNXDllGKj6Aj3zLIhJcLo0Y0w7tXnHIhFxAYuAmfhuGJ0jItmqmt+kzSjgHuB0Vd0vIqmdVbAJridX72BkajxTMpNgbx5U7IAzbnW6LGNMBwRyhj4JKFDVraraACwF5jRrcx2wSFX3A6hqSXDLNJ1hfWEF64squWrKMETEd3aOwHHnOV2aMaYDAgn0wUBhk+Ui/7qmjgOOE5EPRGS1iMxqaUMiskBEckUkt7S0tGMVm6BZ8t5W4mMiuehk/+Hc9Irv6tCENGcLM8Z0SCCBLi2s02bLkcAoYDowD3hYRPoe8SbVxaqapapZKSkp7a3VBNHW0iqWb9jNlVOGkRgbBRWFsOdTG91iTAgLJNCLgKFNlocAu1po8y9VbVTVbcBmfAFvuqm/vL2FaFcE86dl+FasWez7OeZ854oyxhyTQAI9BxglIhkiEg3MBbKbtXkJOBtARJLxdcFsDWahJniKK2p58ZNi5k1KJyUhBvZ8BqsWwcSrIHmk0+UZYzqozUBXVTewEFgJbASWqWqeiNwnIhf6m60EykQkH3gLuENVyzqraHNsFr+zBYAFZ2aC1wuv3Ay9+sLM+xyuzBhzLNoctgigqiuAFc3W3dvkuQK3+h+mGys9WM/SnEIuPnkwg/r2gpxHoCgHLvob9E5yujxjzDGwK0V7mEVvFdDo8fLd6SPh4F5446eQcSaMv9zp0owxx8gCvQfZUFTJE6u2883J6WQkx8EHf4TGapj9AEhLg5mMMaHEAr2H8HiVH764gf7xMdw5aww01MC6J2HcHEi2AUnGhAML9B7iiVXb2VBcyf9eMM437vyz56GuEk79jtOlGWOCxAK9B9hdWctvV27mrONSmH3iQN8kXGuWQOrxkD7V6fKMMUFigR7mGj1ebn9uPW6vcv+cE3xzthSv9V0Veup86zs3JowENGzRhK77X8nng4IyfnPpeNL79/atXLMEohNg/DecLc4YE1R2hh7G/rF6B0+s2sGCMzO5LMs/e0N1GeS9ACfNsznPjQkzFuhh6sOCffwkO49zxqRy16wxvpWeRnjtx+BpgKz5zhZojAk663IJQ8UVtdz49MdkJsfxh7kn4YoQqCqBZdfAzg/hjNsgdYzTZRpjgswCPczUuz1878m1uD3K3646hYTYKN+XoM9eBTXlcMkjcOKlTpdpjOkEFuhh5qcv57O+qJK/XnkKmSnxsO5pePlmiE+D+a/BwPFOl2iM6SQW6GFkWU4hT3+0kxvOGsGssf3h33fBR3/1zdVy6d8hrr/TJRpjOpEFeph49P1t3L88n2kjk7n99CT4x0Ww/T2YuhDO/Sm47FAbE+7sb3mI83qV/1uxkYff38ZXj0/jwbOEyIfPhpp9cNFimGCzKBrTU1igh7BGt4enHnmA1MJPeDo9kSlJ8UQ88Q/onQzfXgmDTnK6RGNMFwoo0EVkFvBHwAU8rKq/bPb6tcBvgGL/qodU9eEg1mmaqa+rZs1D3+baqldpjI4lcr8L2Q8MO913s4p4uwm3MT1Nm4EuIi5gETAT382gc0QkW1XzmzV9VlUXdkKNppnaskJ2/e0SzmjYzKeZCxh/5a8gwq4RM6anC+QMfRJQoKpbAURkKTAHaB7opgvUbPmAhqeuIM1TywdZf+D0C77ldEnGmG4ikNO6wUBhk+Ui/7rmLhGRT0XkeREZ2tKGRGSBiOSKSG5paWkHyu3ZalctIeofF1LhiWHNOc9amBtjviSQQG9pflVttvwyMFxVxwNvAI+3tCFVXayqWaqalZJifbztUZd9O71W3s4H3hMomJPNjDOnO12SMaabCSTQi4CmZ9xDgF1NG6hqmarW+xeXAKcEpzwDUFOyjdiPl7DMOwOdu5RzTx7tdEnGmG4okEDPAUaJSIaIRANzgeymDURkYJPFC4GNwSuxZ1NVlj/nGzA07MJ7OHvcwDbeYYzpqdr8UlRV3SKyEFiJb9jio6qaJyL3Abmqmg18X0QuBNxAOXBtJ9bcozz6wXbG7nmT8oRMJmdNcrocY0w3FtA4dFVdAaxotu7eJs/vAe4JbmkmZ3s5f16Rw5roTUScfIvT5RhjujkbvNxN7auq58anPubi+M9w4UXGzna6JGNMN2eB3g15vcpty9ZTUdvITYM/h4RBMHCi02UZY7o5C/Ru6JH3t/HO56X85LxMEovegdHn2ZWgxpg2WUp0M+sLK/jVq5uYdfwA5iVvhcYaGGPdLcaYtlmgdyMH6xq56ZlPSEuM5VeXjEc2LYeYRBh+htOlGWNCgE2f243c/0o+RftrWHb9VPp4ymHzChj1FYiMdro0Y0wIsDP0buK1vD0syy3iu9NHkBW5DRZPh8ZamHSd06UZY0KEBXo3sK+qnnte2MDxgxK5JTkXHpsFriiY/zqkT3G6PGNMiLAuF4epKve8sIGD9W5emBFJ5PM3QuZ0uPQx6J3kdHnGmBBiZ+gOcnu83PPCBl7P38udXx3NsC1PQVRvuOxxC3NjTLtZoDukut7NdU/ksjSnkIVnj2T+Kf1gw/Nw4mXQq6/T5RljQpB1uTigsLyG7z31MXm7Kvn5RSdwxeRhsOrP4K6FU7/jdHnGmBBlgd6FPF7lsQ+28bvXPidCYMnVWZwzNg28Xsh5GIZOhoHjnS7TGBOiLNC7SEFJFbc9t571hRXMGJPKz75+AoP69vK9uO0dKN8C0+92tkhjTEizQO9kqsozawq575U8ekdH8uC8iVwwfiAiTe7sl/Mw9O4P4+Y4V6gxJuQFFOgiMgv4I74bXDysqr9spd2lwHPAqaqaG7QqQ9SBukbufO5TXs3bw7SRyTzwjQmkJsbC3nx48Xqoq/Q1rNgJ026GyBhnCzbGhLQ2A11EXMAiYCa++4vmiEi2quY3a5cAfB/4qDMKDTWNHi/ffXItH20t557zxnDdGZlERAh4PZB9E1QW+S7rB8g8C6bc6GzBxpiQF8gZ+iSgQFW3AojIUmAOkN+s3f3Ar4Hbg1phiLr/lXw+KCjjN5eO57KsJvfYXvsYFOfCRYthwuXOFWiMCTuBjEMfDBQ2WS7yrztMRCYCQ1X1laNtSEQWiEiuiOSWlpa2u9hQ8Y/VO3hi1Q4WnJn55TA/uAfe+ClknAXjv+FcgcaYsBRIoEsL6/TwiyIRwO+B29rakKouVtUsVc1KSUkJvMoQ8t4XpfwkO48ZY1K5a9aYL7/46j3grofzfw/S0h+rMcZ0XCBdLkVAk9NMhgC7miwnACcAb/tHbgwAskXkwp72xWjO9nIWPLGWUanx/PHy8bje/x2s/jN43b5/AusrYfoPof8Ip0s1xoShQAI9BxglIhlAMTAX+OahF1W1Ekg+tCwibwO397QwX19Ywbcey2Fgn1ievOp4ErLnw8aXYeTM/wZ4fCpMXehsocaYsNVmoKuqW0QWAivxDVt8VFXzROQ+IFdVszu7yO5u054DXP3oGvrFRbH08kEkL50N+zbDV34OU2+07hVjTJcIaBy6qq4AVjRbd28rbacfe1mho2h/DVc/soZeUS6enj+J1H9dBgeK4coXYMTZTpdnjOlB7ErRY1BR08C1j+VQ2+jh+RtOY+iOF2Hnh3DhnyzMjTFdzqbP7aC6Rg/feTyXnWU1LLk6i9EJ9fD6/0D6aXDSlU6XZ4zpgewMvQPW7tjPfS/nsb6okoe+OZEpmf3hxRugvso3JDHC/p00xnQ9C/R22L6vmt++tplXPt1NSkIMD86byPnjB8G2d2H9M3DGbZA6pu0NGWNMJ7BAb0Ojx8vr+Xt5+qOdvF+wj9ioCL5/ziiuPzOTuBj/H987v4bEIXDmHc4Wa4zp0SzQj+JgXSPzlqzms+IDDOoTyy3nHse8SUN9MyYeUrIJtr8H5/4Eono5Vaoxxligt6bB7eWGJ9eyafdB/nD5SVwwYRCuiBbGk+c+Aq5omHhV1xdpjDFNWKC3wOtV7nx+PR8UlPHbyybw9YmDW25YfxDWPQPHXwRxyS23McaYLmKB3ozHq/xseT4vrdvFHV8dzaWnDGm98afLoOEgnHpd1xVojDGtsEBvoriilluWrmPN9nKuPW0435t+lEm0VCHnERgwHoZkdV2RxhjTCgt0vxUbdnP3Pz/F41Ue+MYELj75KGfmADtXQUkeXPCgzdVijOkWenyglxyo495/5fFq3h4mDOnDH+dOZHhy3NHfpArv/x5i+sCJl3VNocYY04YeF+hlVfVsL6uhvLqBraVVPPRWAfVuL3fOGs11Z2QS5QrgKs+NL8MXr8FXfgbRvTu/aGOMCUCPCvTt+6r52oPvUdPgObxuUkYSv7z4RDJT4gPbSN0B+PedMOBEmPzdTqrUGGPar0cF+p/eLMDjVZZcnUVaYgz942MY1CcWaU8f+Fs/990b9PKnwNWj/viMMd1cj0mk7fuqeWldMdeeNpyZ49I6tpHij+Gjv8Gp34EhpwS3QGOMOUYBTQsoIrNEZLOIFIjI3S28foOIbBCRdSLyvoiMC36px+ahtwqIjBCuPyuzYxvwuOGVmyE+Dc75n+AWZ4wxQdBmoIuIC1gEnAeMA+a1ENhPq+qJqnoS8GvggaBXegx2lFXz4ifFXDF5GKkJsW2/oSVrFsPu9XDeLyG2T3ALNMaYIAjkDH0SUKCqW1W1AVgKzGnaQFUPNFmMw3eP+27joTd9Z+c3dPTsvLLI13c+ciaM+3pwizPGmCAJpA99MFDYZLkImNy8kYjcCNwKRAMzWtqQiCwAFgCkp6e3t9YO2VlWwwufFHP11GFfniWxPf59F3g9MPu3dhGRMabbCuQMvaUEO+IMXFUXqeoI4C7gxy1tSFUXq2qWqmalpKS0r9IO+uu7W3CJcMNZR7mM/2g2LYdNr8D0u6Df8KDWZowxwRRIoBcBQ5ssDwF2HaX9UqBb9Evsqazj+dwiLs0aQlpHzs7rq2DFnZA6DqYuDH6BxhgTRIEEeg4wSkQyRCQamAtkN20gIqOaLM4GvgheiR235L2teFS54cwOnp2//Qs4UATn/wFcUcEtzhhjgqzNPnRVdYvIQmAl4AIeVdU8EbkPyFXVbGChiJwLNAL7gWs6s+hAlFc38PRHO7lwwiDS+3fg8vzd62H1n+GUb0H6EV8ZGGNMtxPQhUWqugJY0WzdvU2e/yDIdR2zxz7YRm2j5+hT4LbG64GXb4be/eHc/w1+ccYY0wnC8krRg3WN/P3D7cw6fgCj0hLav4GcR2DXx3DJI9CrX/ALNMaYThDQlaKh5u3NpRysczP/jIz2v/nAbnjzfsg8G064JPjFGWNMJwnLQP9wSxnxMZFMHNq3/W9+9W5w18Ps39mYc2NMSAnLQF+9tYxJGUlEBjK3eVOfvwb5L8FZd0D/Do6MMcYYh4RdoO+prGPbvmqmZvZv3xsbqmH5bZA8Gk7rdt/xGmNMm8LuS9FVW/cBMHVEOwP9nV9B5U64dgVERndCZcYY07nC7gx91ZYy+vSKYuzAxMDfVFUKq/8CE74Jw0/vvOKMMaYThV+gby1jckYSroh2fKH5yRPgaYBpN3deYcYY08nCKtCL9tdQWF7bvu4WrwdyH4OMMyFldOcVZ4wxnYWac0wAAAktSURBVCysAn3VljKgnf3nX7wGlYW+28oZY0wIC69A31pGUlw0x6W24+rQNUsgYSCM/lrnFWaMMV0gbAJdVVm9pYwpmUlEBNp/XrYFtvwHTrnWZlM0xoS8sAn0HWU17Kqsa9/489xHISISTnZ8ckhjjDlmYTMOPWd7OQCTjxbolcXw99m+PnMAr9t3j9DEgV1QoTHGdK6wCfT1RRUkxEQyMiW+9Uav3gUHd8NpN4FE+B4Tr+y6Io0xphOFT6AXVjJ+aJ/W+883/xs2vgzn/C+ccWvXFmeMMV0goD50EZklIptFpEBE7m7h9VtFJF9EPhWR/4jIsOCX2rq6Rg8bdx9gwpBWZldsqIYVd0DKWN/ZuTHGhKE2A11EXMAi4DxgHDBPRMY1a/YJkKWq44HngV8Hu9Cjyd99ALdXmdDadLlv/8LXb36B3RvUGBO+AjlDnwQUqOpWVW0AlgJzmjZQ1bdUtca/uBoYEtwyj259YQUAJ7UU6B8/Aav+7BvJkj6lK8syxpguFUigDwYKmywX+de1Zj7w75ZeEJEFIpIrIrmlpaWBV9mG9YUVDEiMJS0x9r8r3Q2+6XCzb/Jd1v+VnwXt84wxpjsK5EvRlr5l1BYbilwJZAFntfS6qi4GFgNkZWW1uI2OWF9UydRBEfDaj6Gxzrdy18dQvBZO+77vi1BX2Hz/a4wxLQok5YqAoU2WhwC7mjcSkXOBHwFnqWp9cMprW0VNA9v2VfM/gzfDh3+C2L6+4YhRvX03eT7x0q4qxRhjHBVIoOcAo0QkAygG5gLfbNpARCYCfwNmqWpJ0Ks8ik+LKgEYIzt9IX7nVohwdWUJxhjTLbTZh66qbmAhsBLYCCxT1TwRuU9ELvQ3+w0QDzwnIutEJLvTKm5mfWEFIpBaWwApYyzMjTE9VkAdy6q6AljRbN29TZ6fG+S6Ara+qIIRyXFElubbjInGmB4tpCfnUlXWFVYybaAXasog7XinSzLGGMeE9NCPXZV17KuqZ1rCAd8KC3RjTA8W0mfouf4ZFse5/MPkUy3QjTE9V0gH+vNrixiQGMuAuq0QPwDi2jEXujHGhJmQDfRt+6p574t9fHNyOhEledbdYozp8UI20J9cvYPICGFu1kAo3QxpzecLM8aYniUkA722wcNzuYXMOmEAqQ3F4KmHtBOcLssYYxwVkoH+8vpdHKhzc9WUYbA3z7cy1c7QjTE9W8gFuqryxOrtHJcWz6SMJF+giwtSRjtdmjHGOCrkAn1dYQWfFR/gqqnDERFfoCePgsgYp0szxhhHhVygr9lWTkJMJBdN9E/JbiNcjDEGCMFAv/6sEbx319nEx0RC3QGo2GmBbowxhGCgA/TtHe17UrLR99OuEDXGmNAMdAAqi+HVu31fiA6c4HQ1xhjjuNCcnGvHh7DsamishW88AYkDna7IGGMcF9AZuojMEpHNIlIgIne38PqZIvKxiLhFpHPv+bbuaXj8AohJhO/8B8ae36kfZ4wxoaLNQBcRF7AIOA8YB8wTkeZX8ewErgWeDnaBR0jKhONmwXVvQuqYTv84Y4wJFYF0uUwCClR1K4CILAXmAPmHGqjqdv9r3k6o8cvSp/gexhhjviSQLpfBQGGT5SL/unYTkQUikisiuaWlpR3ZhDHGmFYEEujSwjrtyIep6mJVzVLVrJSUlI5swhhjTCsCCfQiYGiT5SHArs4pxxhjTEcFEug5wCgRyRCRaGAukN25ZRljjGmvNgNdVd3AQmAlsBFYpqp5InKfiFwIICKnikgRcBnwNxHJ68yijTHGHCmgC4tUdQWwotm6e5s8z8HXFWOMMcYhoXvpvzHGmC+xQDfGmDAhqh0agXjsHyxSCuzo4NuTgX1BLCdU9MT97on7DD1zv3viPkP793uYqrY47tuxQD8WIpKrqllO19HVeuJ+98R9hp653z1xnyG4+21dLsYYEyYs0I0xJkyEaqAvdroAh/TE/e6J+ww9c7974j5DEPc7JPvQjTHGHClUz9CNMcY0Y4FujDFhIuQCva3b4YUDERkqIm+JyEYRyRORH/jXJ4nI6yLyhf9nP6drDTYRcYnIJyLyin85Q0Q+8u/zs/4J4sKKiPQVkedFZJP/mE/tIcf6Fv/v92ci8oyIxIbb8RaRR0WkREQ+a7KuxWMrPg/6s+1TETm5vZ8XUoEe4O3wwoEbuE1VxwJTgBv9+3k38B9VHQX8x78cbn6AbxK4Q34F/N6/z/uB+Y5U1bn+CLyqqmOACfj2P6yPtYgMBr4PZKnqCYAL30yu4Xa8/w7MarautWN7HjDK/1gA/KW9HxZSgU6T2+GpagNw6HZ4YUVVd6vqx/7nB/H9BR+Mb18f9zd7HPi6MxV2DhEZAswGHvYvCzADeN7fJBz3ORE4E3gEQFUbVLWCMD/WfpFALxGJBHoDuwmz462q7wLlzVa3dmznAE+oz2qgr4gMbM/nhVqgB+12eKFCRIYDE4GPgDRV3Q2+0AdSnausU/wBuBM4dG/a/kCFfwpnCM/jnQmUAo/5u5oeFpE4wvxYq2ox8Ft8N5jfDVQCawn/4w2tH9tjzrdQC/Sg3Q4vFIhIPPBP4GZVPeB0PZ1JRM4HSlR1bdPVLTQNt+MdCZwM/EVVJwLVhFn3Skv8/cZzgAxgEBCHr8uhuXA73kdzzL/voRboPeZ2eCIShS/Mn1LVF/yr9x76L5j/Z4lT9XWC04ELRWQ7vq60GfjO2Pv6/0sO4Xm8i4AiVf3Iv/w8voAP52MNcC6wTVVLVbUReAE4jfA/3tD6sT3mfAu1QO8Rt8Pz9x0/AmxU1QeavJQNXON/fg3wr66urbOo6j2qOkRVh+M7rm+q6hXAW8Cl/mZhtc8AqroHKBSR0f5V5wD5hPGx9tsJTBGR3v7f90P7HdbH26+1Y5sNXO0f7TIFqDzUNRMwVQ2pB/A14HNgC/Ajp+vppH2chu+/Wp8C6/yPr+HrU/4P8IX/Z5LTtXbS/k8HXvE/zwTWAAXAc0CM0/V1wv6eBOT6j/dLQL+ecKyBnwKbgM+AfwAx4Xa8gWfwfUfQiO8MfH5rxxZfl8sif7ZtwDcCqF2fZ5f+G2NMmAi1LhdjjDGtsEA3xpgwYYFujDFhwgLdGGPChAW6McaECQt0Y4wJExboxhgTJv4f2woKOp2d3X0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(train_scores2)\n",
    "plt.plot(test_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Momentum 效果对比"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3dd3xV9f3H8dcngRBWCCusJAzZIDICLkRxoqio2ArUgVrRto46q9WftXRoba1ipa1UcSIIuBAZVtQqygoiK5AQRgYjhBVGIMnN/fz+OBcIIZAbuDcn997P8/HII/eM3Ps+nPjx5Hu+5/sVVcUYY0zoi3I7gDHGmMCwgm6MMWHCCroxxoQJK+jGGBMmrKAbY0yYqOXWBzdr1kzbtWvn1scbY0xIWrp06Q5VbV7RNtcKert27UhNTXXr440xJiSJSNaJtlmTizHGhAkr6MYYEyasoBtjTJjwq6CLyBARSReRTBF5vILtbUVknoisEJGvRSQx8FGNMcacTKUFXUSigfHAlUB3YKSIdC+329+At1W1FzAWeDbQQY0xxpycP1foA4BMVd2gqsXAFGBYuX26A/N8r7+qYLsxxpgg86egtwFyyizn+taVtRwY7nt9PdBQRJqWfyMRGSMiqSKSmp+ffyp5jTHGnIA//dClgnXlx9x9BHhFREYD3wCbAc9xP6Q6AZgAkJKSYuP2GlNd9m2DH96G0hK3kxiALkOgTb+Av60/BT0XSCqznAhsKbuDqm4BbgAQkQbAcFUtCFRIY8xpUIXpd0DWd1R8fWaqXcOWrhX0JUAnEWmPc+U9AhhVdgcRaQbsUlUv8AQwMdBBjTGnaMX7TjG/5mXod5vbaUwQVdqGrqoe4F5gLrAGmKqqq0VkrIhc69vtIiBdRDKAFsCfgpTXGFMVB/fA509BYn/oc4vbaUyQ+TWWi6rOAmaVW/d0mdfTgemBjWaMOW1f/hEKd8LNH0CUPUcY7uwMGxOuNi+F1Neh/13Q6iy305hqYAXdmHCUOQ/evh4atICLn3Q7jakmVtCNCSeqsPBfMOlGiE+CO+ZCbCO3U5lq4tp46MaENK8Xvn4Wlr0D6nU7zVHeUijcAV2vhutfhToN3E4UVlSVf369nncWZFGqp/4ozeNDujK8X+CHvLKCbkxVFe2DD+6CjNnQ6QqIa+V2omMldHfaze0maEAdLC7l0enLmbliKxd0akZi43qn/F5tGtcNYLKjrKAbU5H8DNhcwYxa6oXvX4EdGXDl8zBgDIg9rFNTqCrfrNtB/r6igL/v2wuyWLWlgN8M6co9F3ZAauB5t4JuTHkrpsEnv4LSExSF2HinG+AZg6s3lzmpIk8pT320imlLc4Py/g3q1OI/t6RwafcWQXn/QLCCbsxhXi98ORbmvwhtz4ehf4fascfvV6+ZtU3XMPn7ivjFu0tJzdrNfRd35KcpSZX/UBU1rh9Dgzo1u2TW7HTGHLZ8Cnz/j+AOLlVyEAqyod/tTnNKrZjgfVYN9H3mDp6bs5bC4lK3o1RZ/r4iijylvDKqD1f3au12HNdYQTc1m7cUvvidU8xb9oKEbsH9vAsfgz43R1y7+DsLNvHMp2kkNa5Lj9ah182xZ+s4fn5BB3q2Cb3sgWQF3dRchwrgg5/Dus+dXhtDnoXo2m6nCiueUi/PfLqadxdmc0nXBF4a0ZuGsfZvHKqsoJuaaed6mDwSdq132rL73+l2orD0fmoO7y7M5u5BHXhsSFeioyLrL5NwYwXd1DwbvoaptznNHrd8DO0vcDtRWFJV3l2YTfdWcTx+Zdca2Q3PVI0VdOMuVad9fO1nh1dAbio06wwjJ0OT9q7GC2c/5uxhzda9/PG6nlbMw4QVdOOe4kKYcS+s+gBa9T465kjfW+GysRAb526+MDdpUTb1Y6K5rk/5KYJNqLKCbqrPgR2wM9N5XVoMn/8fbF0Olz4D5/864nqWuKmgsISZK7ZwfZ/EGt+32vjPrzMpIkOAcUA08JqqPlduezLwFhDv2+dx36QYxjjWfwXTRsOhPUfXxTRwmlW6XOlarEj14bJcDpV4+dnZyW5HMQFUaUEXkWhgPHAZzoTRS0RkhqqmldntKZyp6f4lIt1xZjdqF4S8JtSowuIJMOcJp138hglHux426wKN7M/96qaqvLcom7OS4iO+33a48ecKfQCQqaobAERkCjAMKFvQFTjc4NkI2BLIkCaEzX0SFo6Hzlc6xdzaxV33xZrtrNu+n+eH93I7igkwfwp6GyCnzHIucHa5fZ4BPheR+4D6wKUVvZGIjAHGACQn2596YW/D104xT7kDrnrBhnOtAd5blM3Tn6yiU0IDrj6rhg37a06bPwW9ojtV5Ud2Hwm8qaoviMi5wDsi0lP12JH/VXUCMAEgJSXl1EeHNzWfpxhmPQqN28EVf7Zi7oKSUi9ZOw9weB6Gdxdm8daCLC7s3Jx/jOpDvRi7GRpu/DmjuUDZocsSOb5J5U5gCICqLhCRWKAZsD0QIU0IWuAbM3zUNKgdnMH8zck9N3str8/feMy6uy5oz+NXdrMnQsOUPwV9CdBJRNoDm4ERwKhy+2QDlwBvikg3IBbID2RQE0L2ZMP/nnemQet8udtpIlJhsYepqTkM6tycn6Y4U521jIslpV0Tl5OZYKq0oKuqR0TuBebidEmcqKqrRWQskKqqM4CHgf+IyIM4zTGjVU9jwj0TukpL4LOHnT7lQ551O03Emrl8K/sOebjv4o70tyIeMfxqRPP1KZ9Vbt3TZV6nAecHNpoJOQd2wtRbIWu+M554vN34dsukxdl0SmhAStvGbkcx1cjuVJnAyEuD/wyG3CVw/QQ4+263E0WsVZsLWJ6zh5+dnWxjtEQYu81tTl/JQXhzKETHwO2zIDHF7UQR7b3F2cTWjuL6voluRzHVzAq6OX0bv4GDu5yJk62Yu2p/kYdPlm3mml6taVTXJqqINNbkYk5f+iyIaQjtbNxyt328bDMHiksZZWO0RCQr6Ob0eL2QPgc6Xgy16ridJqLtL/Lwjy/XcVZiI3onxbsdx7jACro5PVt/hP3boMtVbieJeOO+yCBvbxG/u7aH3QyNUFbQzelJnw0SBZ3sASI3pW/bx8TvNjGifxJ9k62rYqSygm5OT/psSDoH6tnDK25RVf7v41XExdbiN0O6uh3HuMh6uZhTtycH8lbCZX9wO0nE+XZdPj9kOZOFbNt7kMWbdvHcDWfSuH6My8mMm6ygm1OXMcf5bu3n1cbrVcbNW8e4eeuOWT+4S3N+mpJ0gp8ykcIKujl16bOgaUdo1tHtJBGhsNjDw1OXM3vVNob3TeRP1/ckJtppNRXBboQaK+jmFB3cA5vm2yP+1WTznoPc9VYqa7ft5cmruvHzC9pbATfHsYJuTs2K96G0GHre6HaSsJe6aRd3v7OUYo+X12/rz+CuCW5HMjWUFXRTdaqQ+ga07gute7udJqxNTc3hyY9W0ia+Lq/dlkLHhIZuRzI1mBV0U3XZCyF/DVz7D7eThC1PqZdnfTMOnd+xKeNH9SW+nvVgMSdnBd1U3dI3oE4c9BzudpKwVHCwhPsmL+ObjHxGn9eOp4Z2o1a0PTJiKudXQReRIcA4nBmLXlPV58ptfxEY7FusBySoqg0mEY4Kd8Hqj6HvrRBT3+00YcdT6uWmVxewPn8/z95wJiMH2CBbxn+VFnQRiQbGA5fhTBi9RERm+GYpAkBVHyyz/31AnyBkNTXBj+9BaRGk3O52krA0b+121m7bx7gRvRnWu43bcUyI8efvuAFApqpuUNViYAow7CT7jwQmByKcqWFUneaWpHOgRQ+304SlSYuyadUolqFntnI7iglB/hT0NkBOmeVc37rjiEhboD3w5Qm2jxGRVBFJzc/Pr2pW47ZVH8DOTOh/p9tJwlL2zkK+XZfPTf2TrM3cnBJ/fmsqenpBT7DvCGC6qpZWtFFVJ6hqiqqmNG/e3N+MpiY4tBfmPgmt+9jN0CCZvCSbKBFG9Ld2c3Nq/CnouUDZQSISgS0n2HcE1twSnr5+FvbnwdAXICra7TRhp9jjZVpqDhd3TaBlo1i345gQ5U9BXwJ0EpH2IhKDU7RnlN9JRLoAjYEFgY1oXLdtJSx6FfqNhjb93E4Tlj5P28aO/cX8zKaOM6eh0oKuqh7gXmAusAaYqqqrRWSsiFxbZteRwBRVPVFzjAlFXi989jDUjYdLnnY7TdiatDCbxMZ1GdTJmiLNqfOrH7qqzgJmlVv3dLnlZwIXy9QYWd9BziK45mWbxCJIlmbtYsGGnTx6RReiomzALXPq7Fa6OblN850p5npc53aSsOQp9fLkR6to1SiW0ee1czuOCXFW0M3JZX0HLc+E2EZuJwlLby/IYu22fTx9dXfq17GROMzpsYJuTsxTDLlLoO35bicJS3l7D/H3/2YwqHNzhvRs6XYcEwasoJsT27IMPIeg7XluJwlLf/psDcWlXsZe28MmqzABYQXdnFjWfOd78rnu5ghDqzYXMGP5Fu4Z1IF2zWyQMxMYVtDNiWV9D827Qv1mbicJO5MWZRNbO4o7B3ZwO4oJI1bQTcVKPZC9yJpbgmDfoRI++XEzV/dqTaN6td2OY8KIFXRTsbyVULzPbogGwSc/bqGwuNSeCjUBZwXdVCzre+e7XaEHlKoyaVE23VvF0TvJ5oAxgWUF3VQs63to3B7iWrudJKz8mLOHNVv3MursZOvZYgLOCro5ntfrFHRrbgm4SYuyqR8TzXV9bDYiE3hW0M3xdqTDwV3W3BJgBYUlzFyxhWF92tDAngo1QWAF3Rwvc57zvd1Ad3OEmRe/yKDI4+XWc9u6HcWEKSvo5njpsyGhBzS2whMoqzYX8PaCTdx8dlu6toxzO44JU1bQzbEKd0H2AuhypdtJwobXqzz18Soa14vhkcu7uB3HhDEr6OZYmV+AllpBD6CpqTn8mLOHJ67qZg8SmaCyOzPmWOmzoH4CtO7rdpKQkrl9P9OX5lLq9R63bdrSXPq3a8zwvtazxQSXXwVdRIYA44Bo4DVVfa6CfX4KPAMosFxVRwUwp6kOnmJY9wX0vB6i7I83f321djv3TV7GoZJSYmod/+/WpH4Mf7zuTOt3boKu0oIuItHAeOAyIBdYIiIzVDWtzD6dgCeA81V1t4gkBCuwCaKs75zH/btc5XaSkKCqTPhmA8/NWUuP1nFMuCWF1vF13Y5lIpg/V+gDgExV3QAgIlOAYUBamX3uAsar6m4AVd0e6KCmGqTPhlqx0P5Ct5PUeIdKSvntRyv58IfNDO3Vir/deBZ1Y6LdjmUinD8FvQ2QU2Y5Fzi73D6dAUTkO5xmmWdUdU75NxKRMcAYgORkG5ioRlF1CnqHwRBTz+00Ndr2vYe4+92lLMvew4OXdub+Szpac4qpEfwp6BX9pmoF79MJuAhIBL4VkZ6quueYH1KdAEwASElJKf8exk15q6EgGwY94naSGmd5zh7mZ+4AnC6IkxZlU3CwhH/f3JchPVu5nM6Yo/wp6LlAUpnlRGBLBfssVNUSYKOIpOMU+CUBSWmC77uXIDoGOg9xO0mN8t6ibJ7+ZBUe79Hrj+Qm9Zj+i3Pp0domzjY1iz8FfQnQSUTaA5uBEUD5HiwfAyOBN0WkGU4TzIZABjVBtPEbWDkNLvwNNGzhdpoaoaTUyx9npvHWgiwu7NycF2/qfWT8lVpRQlSUNbGYmqfSgq6qHhG5F5iL0z4+UVVXi8hYIFVVZ/i2XS4iaUAp8Kiq7gxmcBMgnmL47BGIbwsDH3Q7jWtUlfun/MhXa537+R6vl0MlXn4+sD1PXNWNaCvgJgT41Q9dVWcBs8qte7rMawUe8n2ZULLwn87oiqOmQu3I7XL30bLNfLp8C1f3akWLuFgA+rdrbG3kJqTYk6KRrCAX/vcX6Ho1dL7C7TSuKThYwp9nraFPcjwvj+hjzSkmZFlBj2TfjQOvB4Y863YSV73weTq7DhTz5u0DrJibkGbPd0eq4gOwfAr0uB7iI/eZgFWbC3h3YRa3ntuOnm2s14oJbVbQI9WqD6BoL/S73e0krtm+7xCPTFtOk/p1eOjyzm7HMea0WZNLpEp9A5p3g+Rz3E7iilWbC7jr7VT2FJbwr5v7Ehdrw9qa0GdX6JFoy4+w5QdIuR0i8JH1WSu3cuO/v0eA6b84l4u62FhyJjzYFXokWvoG1KoLvW5yO0m1y8jbx/2Tl9ErsRGv3pJC84Z13I5kTMBYQY80h/bCimnQczjUjXc7TbVSdaaCaxBbi9du60+T+jFuRzImoKzJJZKoOl0VSw44zS0R5uMfN7N44y4eu6KrFXMTluwKPVJ4iuCzh2HZO9DjBmjTz+1E1argYAl/+mwtZyXFM6J/UuU/YEwIsoIeCfbnw9RbIHsBDHoULvptxN0MffG/Gew6UMSbt/e3h4dM2LKCHu62rYTJI+FAPtw40Wk7jyBerzJu3jre/H4Tt57b1h4eMmHNCno4W/MpfDgGYuPhjjnQuo/biapVYbGHR6YtZ9bKbQzvm8iTQ7u5HcmYoLKCHo5U4Zu/wld/gjYpMGISNGzpdqpqtXnPQe56K5W12/by1NBu3DmwvU0TZ8KeFfRwU1wIn/wSVn/k9DO/5mWoHet2qmqVumkX97y7lKISL6+P7s9ge3DIRAgr6KGucBfs3uS89hTBnN/A1hVw6e/h/Aci7ubn1NQcnvxoJW3i6zJlTAodExq6HcmYauNXQReRIcA4nBmLXlPV58ptHw38FWeKOoBXVPW1AOY0FVn/FUwbDYfKzMUd0xBGvR9x45t7Sr08O3str8/fyMCOzXhlVB/i61lfcxNZKi3oIhINjAcuw5kMeomIzFDVtHK7vq+q9wYhoylPFRa9CnN/C827wLBXIMo3uFTLntAo0d181azgYAn3TV7GNxn5jD6vHU8N7UataHtmzkQef67QBwCZqroBQESmAMOA8gXdVAdPMcx6GH54G7oMhRtehTqR1aywZc9BXvg8g8JiDwCrt+xly56DPHvDmYwcELljuxvjT0FvA+SUWc4Fzq5gv+EiMgjIAB5U1ZzyO4jIGGAMQHKy/YdXZQd2wPu3QPb3cMEjMPhJiIqsK1FV5fEPV7Jow07aNq0HQOP6MTx/Yy/O6dDU5XTGuMufgl7RXTUtt/wpMFlVi0TkHuAt4OLjfkh1AjABICUlpfx7mJPZtsr3gNB2GP46nHmj24mCwutVDpaUUr9Oxb+ac1Zt45uMfJ6+ujt3DGxfzemMqdn8ubzLBcoOfpEIbCm7g6ruVNUi3+J/gMgaKCTYPEXw9rXgLYHbZ4VtMd+xv4ibJizg7D/P48u1ecdtP1DkYezMNLq1iuPWc9u6kNCYms2fgr4E6CQi7UUkBhgBzCi7g4i0KrN4LbAmcBENm+ZD4U64+sWwHVRr9ZYChr3yHSs3F9CqUSx3vpXKv/+3HtWjf8i9/OU6thYc4o/X9bSbnsZUoNImF1X1iMi9wFycbosTVXW1iIwFUlV1BnC/iFwLeIBdwOggZo486bOdCSk6XOR2kqCYs2orD76/nPh6tZl293l0TGjAI9OX89zstXyXuYPExnXxeuGDH3L5aUoi/do2djuyMTWSlL0Cqk4pKSmamprqymeHFFV46Uxo2QtGvud2moBSVV6el8mLX2TQJzmeV2/pR0LD2CPb/vn1et5dmEWp1/kdbR1fl9dvS6FpA5tlyEQuEVmqqikVbbMnRWu6vFVQkAMXPuZ2klPiKfVyoKj0uPXFpV6embGaz1Zu5Ya+bfjz9WcSWzv6yHYR4VeDO/KrwR2rM64xIc0Kek2XPhsQ6DzE7SRVtnpLAXe/s5Tc3Qcr3C4Cv72qK3dd0MEGzjImAKyg13Tps50boQ1Ca4Cp2Su38tBUp138qaHdiKqgYPdKbERKuyYupDMmPFlBr8n2boUtP8DF/1flH121uYCZK7ZWuC2xcV1GDkgmuszMPbm7C5m6JIfiUv/vqdSKEm7sl0i7ZvWPrPN6lZe/XMdLX6w7rl3cGBNcVtBrsow5zvcuV1XpxwoOljD6jcXsLiw5pmgDoE779Vdrt/PSiN40jK3N4o3OcLN7Cour1B3QU+rl7QWbGP+zvlzQqfkxE0rc0KcNf77h2HZxY0xwWUGvyTLmQHwyJFRtpp0XPk9n14FiZtw78Lgp11SVdxZm8ftP0xj+r++5oW8iL3yeTlLjeky751zOaN7A78/J3lnIXW+nMvqNJTx0WWdmrdxK2ta91i5ujEvs6YyaauO3zvC4na+s0pjmK3MLeGdhFree267C+TNFhFvPbcfbdwwgb28Rz81eyzkdmvLRL8+vUjEHSG5ajw9+eR6DuyTw17npZO8sZOJt/Rkz6Awr5sa4wPqh10RLXofZj0GTM+CWD/0eDtfrVa7/1/ds3n2QLx+5kLjY2ifdP3tnIQs27GB438TTevLS61U+XLaZPsnxVf6fgjGmaqwfeqhQdQr54gnQ6XIY/hrE+j9L/aTF2SzP2cNLN/WutJiDc4Wd3PT0R72M8t0cNca4ywp6TbLha6eYD7gbhjwLUf7dUFRVXvt2I8/OXsPAjs0Y1rt1cHMaY2okK+g1SepEqNsELhvrdzEv8pTy2w9X8cEPuQw9sxV//Ukva782JkLZTdGaYt82vGtnMa10EBMXbcWfexuqyl1vL+WDH3J58NLOvDKqD/Vi7P/RxkQqK+g1xPIZrxClHt44dCFjZ6bxmw9WUOQ5fgyUsj5atplvMvL5/bU9eODSTnZlbkyEs8s5l3m9yh8+XcmdGZNZHdubyQ/cwuvzN/Dyl5lsyD/AcN/NxiiBi7ok0CLOeeqy4GAJf561hj7J8dxyjk32YIyxgu66SYuz2bhoBokxO2h1zd+Irlebhy7vQqcWDXls+gpSs1Ye2bdZgxj+fXM/Uto1OfLw0Ju3DyCq/NOgxpiIZAXdRTv2F/HXOWt5I+5bNDqB6K5Dj2y75qzWXNw1gX2HnJnt8/Ye4oEpyxj5n4WMGdSBdxdmccs5bSt8eMgYE5n8akMXkSEiki4imSLy+En2u1FEVEQq7PRufLavhVcHwbjezNR76XtoEdLnZqgVc8xu9evUomWjWFo2iuWspHg+/tX5nN2+KeO/Wk+T+nV46PIuLh2AMaYmqvQKXUSigfHAZTgTRi8RkRmqmlZuv4bA/cCiYAQNG6ow89d4dmXxzaGedG7RAElqDuf8otIfja8Xw5u39+e1+RvpnRRPo7qVPzxkjIkc/jS5DAAyVXUDgIhMAYYBaeX2+wPwPPBIQBOGiVKv8u26fBqvm85Z2QsYV/d+Pqw/mP+OGQRV6GpYKzqKey48I4hJjTGhyp8mlzZATpnlXN+6I0SkD5CkqjNP9kYiMkZEUkUkNT8/v8phQ9mL/83g/je+os2SZ1nq7cQ/9wzgd9d0t37jxpiA8aeaVNSF4shTLyISBbwIjK7sjVR1AjABnMG5/IsY+tbn72fCNxt4veVnNC3Yz/7h0/g28Sxax9d1O5oxJoz4U9BzgaQyy4nAljLLDYGewNe+B1taAjNE5FpVDcvhFL1eZfxXmfx3TV6F21s1imXssJ60yJmDLhiP5O3ho9oeuu/ZiJx9D+16nlPNiY0xkcCfgr4E6CQi7YHNwAhg1OGNqloANDu8LCJfA4+EazE/UOThwfd/5PO0PPq1bUxc7LH/hArMX7edmS/9iju909nf8Aw2HWpI5xYNkaTzYfBv3QlujAl7lRZ0VfWIyL3AXCAamKiqq0VkLJCqqjOCHbKmyNnlzNCTkbeP313TndHntTv+cfviA+ydfCdxG2cz3XsRz+8fQ0JCHJ/8cqDzuKcxxgSJX3fkVHUWMKvcuqdPsO9Fpx+rZnry41Vs3nOQN28fwKDOzSve6bNHiNs0lwMXjWXq2v7syNrNq8N6Hj+3pzHGBJh1sfBTscfL4o07GTkg+cTFPOt7WP4eDHyI+hc9wHsXeMnbV0Qbu/lpjKkGNtqin1ZuLuBQiZez2zepeIfSEvjsYWiUBIOcrvi1oqOsmBtjqo1doftp0cadAPRvd4KCvuhV2J4GN02CmPrVmMwYYxx2he6nxRt30TGhAU0b1Dl+494t8PWzzjygZQbYMsaY6mQF3Q+lXiV10+4TN7f873mnyeXKv4BNMmGMcYkVdD+s2bqX/UUeBlRU0A/thRVT4cyfQJMO1R/OGGN8rKD7YdHGXQAVF/SVU6HkAKTcUc2pjDHmWFbQ/bB4406Sm9SjVaNyPVZUIfUNaHkmtOnrTjhjjPGxgl4JVWXxxl0VX53npkLeKufq3NrOjTEus4Jeiczt+9ldWFJxQU+dCDENnPZzY4xxmRX0ShxuPz+uh8vB3bD6Q6eY12noQjJjjDmWFfRKLNq4ixZxdUhuUu/YDT9OBs8hSLndnWDGGFOOFfSTWJ+/nzmrtnJx1xbHjqp4cDd8+wIknwutznIvoDHGlGEF/QRUld99sprY2tE8eFmnYzd++Uc4uMt5kMgYY2oIK+gn8NnKrczP3MEjl3choWHs0Q1blsGS16H/XXZ1boypUaygV2B/kYc/zEyjR+s4bj6n7dEN3lKY+RDUbw4XP+leQGOMqYCNtliBcV9ksH1fEf++ud+xE1P88BZs+QGunwCxjdwLaIwxFfDrCl1EhohIuohkisjjFWy/R0RWisiPIjJfRLoHPmr12FNYzFsLshjeN5E+yY2PbjiwA774PbQdCL1+6l5AY4w5gUoLuohEA+OBK4HuwMgKCvZ7qnqmqvYGngf+HvCk1eSDHzZT7PFyx/ntj93wxe+geD8M/Zs9FWqMqZH8uUIfAGSq6gZVLQamAMPK7qCqe8ss1gc0cBGrj6oyaVEWfZLj6d467uiG7IWw7F0455eQ0M29gMYYcxL+FPQ2QE6Z5VzfumOIyK9EZD3OFfr9Fb2RiIwRkVQRSc3Pzz+VvEG1aOMuNuQfYNSA5KMrSz3O1HJxbeDC37gXzhhjKuFPQa+ofeG4K3BVHa+qZwC/AZ6q6I1UdYKqpqhqSvPmJ5ho2UWTFmUTF1uLq3u1PrpyyX+cAbiGPAt1GrgXzhhjKuFPQc8FksosJwJbTrL/FOC60wnlhh37i5izaivD+yVSNybaWblvG3z5JzjjEuh2rbsBjTGmEk+4TDEAAAufSURBVP4U9CVAJxFpLyIxwAhgRtkdRKTso5RDgXWBi1g9pi/NpaRU+dnZZZpb5j4JpUVw1V/tRqgxpsartB+6qnpE5F5gLhANTFTV1SIyFkhV1RnAvSJyKVAC7AZuC2boQCv2eJm0KIsB7ZvQMcE3cuKG/8Gq6TDoMWh6hrsBjTHGD349WKSqs4BZ5dY9Xeb1AwHOVa3+8+0GcnYdZOywns4KTzHMegTi28IFD7kbzhhj/BTxT4rm7i7kH1+u44oeLRjcJcFZuXA87MiAUVOhdt2Tv4ExxtQQET+Wy9hP0xCEp6/p4azYkwP/ex66DIXOV7gbzhhjqiCiC/qXa/P4PC2P+y/pRJt435X4nMedyZ+vfM7dcMYYU0URW9BLSr08MyONjgkNuHOg7zH/jM9h7Uy48FGITz75GxhjTA0TsQV93prtZO8q5NEruhBTKwpKDsLsR6FpJzj3PrfjGWNMlUXsTdH3FmfTqlEsl3T13Qid/xLs3gS3fgK1YlzNZowxpyIir9CzdxbyTUY+N/VPolZ0lDNH6HfjoMf10OEit+MZY8wpiciCPnlJNtFRwoj+vnby5e+D5yAMfNDdYMYYcxoirqAXe7xMS83h4q4JtGwU6/RoWfoGtO5rc4QaY0JaxBX0z9O2sWN/8dExW7IXQP5aSLnD3WDGGHOaIq6gv7com8TGdRnUyTd8b+obUCcOet7gbjBjjDlNEVPQvV7l75+n8/36nYw6O5moKIEDOyHtYzhrBMTUdzuiMcaclojotnigyMNDU39k7uo8bkpJ4ucDOzgblr8HpcXQ73Z3AxpjTACEfUE/VFLKT/69gLXb9vL7K8/g1n0TkCnZzsYtP0DSOdCi/JzXxhgTesK+yeWH7N2kbd3LX4b34jb9GEl9HQ7kQ+FOaNweLnrc7YjGGBMQYX+FnrFtHwAXJxyA2X+HnsPhxokupzLGmMDz6wpdRIaISLqIZIrIcZe0IvKQiKSJyAoRmScibQMf9dSk5+2jcd1aNPn2/yA6Bi7/k9uRjDEmKCot6CISDYwHrgS6AyNFpHyj8zIgRVV7AdOB5wMd9FSlb9vHz+JXIus+h8FPQFwrtyMZY0xQ+HOFPgDIVNUNqloMTAGGld1BVb9S1ULf4kIgMbAxT42qsjMvl5/vnwAJPWDA3W5HMsaYoPGnoLcBcsos5/rWncidwOyKNojIGBFJFZHU/Px8/1Oeou0Zi3mPJ2hQWgDXvATRYX/LwBgTwfwp6FLBOq1wR5GbgRTgrxVtV9UJqpqiqinNmzf3P6W/DuyEgs3O18rpNJs6DEHJGDoNkgYE/vOMMaYG8eeSNRdIKrOcCGwpv5OIXAo8CVyoqkWBiVcFm76DN686ZlV+3JkM2/4L5vU4t9rjGGNMdfOnoC8BOolIe2AzMAIYVXYHEekDvAoMUdXtAU/pj5xFzvehf4eoWlC7Hn9blUTtogPExdZ2JZIxxlSnSgu6qnpE5F5gLhANTFTV1SIyFkhV1Rk4TSwNgGkiApCtqtcGMffxtqdBoyTof+eRVau/+pbOLRtWawxjjHGLX3cJVXUWMKvcuqfLvL40wLmqbvsaSOh2ZNFT6mX99v0M6tzMxVDGGFN9wuPR/9IS2JFxTEHftPMAxaVeurSwK3RjTGQIj4K+c70zamJCjyOr0rftB6CzFXRjTIQIj4K+Pc35XuYKPX3bXqIEOiY0cCmUMcZUr/Ap6BINzTofWZWet492zeoTWzvaxWDGGFN9wqSgr4GmZ0Dt2COrMvL209V6uBhjIkiYFPS0Y5pbDhaXsmnnAWs/N8ZElNAv6MUHYNfGY2+I5u1DFevhYoyJKKFf0PPTAT3mCv2bjHxEIKVdE/dyGWNMNQv9gr59jfM94egQ7V+syaN3UjzNG9ZxKZQxxlS/MCjoaVArFpq0ByBv7yFW5BZwabcWLgczxpjqFR4FvXkXiHK6J85b44wNZgXdGBNpQr+g56Udc0N03po8kprUpXMLe6DIGBNZQrugF+6C/duO3BAtLPYwP3MHl3ZrgW/UR2OMiRghV9B37i9i6pIcVPW4G6Lz1+2gyOO15hZjTEQKuUk23/p+Ey9/mUnW+tU8vPN3REXHQKuzAKf9vGFsLQa0t+6KxpjIE3IF/deXdqbV7lSuSHuMwijw/mQycQ2a4/Uq89Zu56IuCdSODrk/PIwx5rT5VdBFZAgwDmfGotdU9bly2wcBLwG9gBGqOj3QQQ+LWjGZken3s7dhMtftuZ+d0700a/A/Sr3Kjv1FXNotIVgfbYwxNVqlBV1EooHxwGU4E0YvEZEZqppWZrdsYDTwSDBCHqPJGdB5CHHX/ZMX8pWJ322kpNQLQEq7xlzW3drPjTGRyZ8r9AFApqpuABCRKcAw4EhBV9VNvm3eIGQ8VvLZkDwJgLOSYNyIPkH/SGOMCQX+NDa3AXLKLOf61lWZiIwRkVQRSc3Pzz+VtzDGGHMC/hT0ijp066l8mKpOUNUUVU1p3rz5qbyFMcaYE/CnoOcCSWWWE4EtwYljjDHmVPlT0JcAnUSkvYjEACOAGcGNZYwxpqoqLeiq6gHuBeYCa4CpqrpaRMaKyLUAItJfRHKBnwCvisjqYIY2xhhzPL/6oavqLGBWuXVPl3m9BKcpxhhjjEvskUpjjAkTVtCNMSZMiOop9UA8/Q8WyQeyTvHHmwE7AhgnVETicUfiMUNkHnckHjNU/bjbqmqF/b5dK+inQ0RSVTXF7RzVLRKPOxKPGSLzuCPxmCGwx21NLsYYEyasoBtjTJgI1YI+we0ALonE447EY4bIPO5IPGYI4HGHZBu6McaY44XqFboxxphyrKAbY0yYCLmCLiJDRCRdRDJF5HG38wSDiCSJyFciskZEVovIA771TUTkvyKyzve9sdtZA01EokVkmYjM9C23F5FFvmN+3zdAXFgRkXgRmS4ia33n/NwIOdcP+n6/V4nIZBGJDbfzLSITRWS7iKwqs67CcyuOl321bYWI9K3q54VUQS8zHd6VQHdgpIh0dzdVUHiAh1W1G3AO8CvfcT4OzFPVTsA833K4eQBnELjD/gK86Dvm3cCdrqQKrnHAHFXtCpyFc/xhfa5FpA1wP5Ciqj1x5iseQfid7zeBIeXWnejcXgl08n2NAf5V1Q8LqYJOmenwVLUYODwdXlhR1a2q+oPv9T6c/8Db4BzrW77d3gKucydhcIhIIjAUeM23LMDFwOFJx8PxmOOAQcDrAKparKp7CPNz7VMLqCsitYB6wFbC7Hyr6jfArnKrT3RuhwFvq2MhEC8iraryeaFW0AM2HV6oEJF2QB9gEdBCVbeCU/SBBPeSBcVLwGPA4blpmwJ7fEM4Q3ie7w5APvCGr6npNRGpT5ifa1XdDPwNZ4L5rUABsJTwP99w4nN72vUt1Ap6wKbDCwUi0gD4APi1qu51O08wicjVwHZVXVp2dQW7htv5rgX0Bf6lqn2AA4RZ80pFfO3Gw4D2QGugPk6TQ3nhdr5P5rR/30OtoEfMdHgiUhunmE9S1Q99q/MO/wnm+77drXxBcD5wrYhswmlKuxjnij3e9yc5hOf5zgVyVXWRb3k6ToEP53MNcCmwUVXzVbUE+BA4j/A/33Dic3va9S3UCnpETIfnazt+HVijqn8vs2kGcJvv9W3AJ9WdLVhU9QlVTVTVdjjn9UtV/RnwFXCjb7ewOmYAVd0G5IhIF9+qS4A0wvhc+2QD54hIPd/v++HjDuvz7XOiczsDuNXX2+UcoOBw04zfVDWkvoCrgAxgPfCk23mCdIwDcf7UWgH86Pu6CqdNeR6wzve9idtZg3T8FwEzfa87AIuBTGAaUMftfEE43t5Aqu98fww0joRzDfweWAusAt4B6oTb+QYm49wjKMG5Ar/zROcWp8llvK+2rcTpAVSlz7NH/40xJkyEWpOLMcaYE7CCbowxYcIKujHGhAkr6MYYEyasoBtjTJiwgm6MMWHCCroxxoSJ/wf837Bw+Mx6TQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(test_scores1)\n",
    "plt.plot(test_scores2)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
